ai-research-survey

scan.json (26850B)

---

 {
   "paper": {
     "title": "AdvEvo-MARL: Shaping Internalized Safety Through Adversarial Co-Evolution in Multi-Agent Reinforcement Learning",
     "authors": [
       "Zhenyu Pan",
       "Yiting Zhang",
       "Zhuo Liu",
       "Yolo Yunlong Tang",
       "Zeliang Zhang",
       "Haozheng Luo",
       "Yuwei Han",
       "Jianshu Zhang",
       "Dennis Wu",
       "Hong-Yu Chen",
       "Haoran Lu",
       "Haoyang Fang",
       "Manling Li",
       "Chenliang Xu",
       "Philip S. Yu",
       "Han Liu"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2510.01586"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided anywhere in the paper. There is no mention of releasing code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: MATH-500, JailbreakBench, Wild Jailbreak, Strong Reject, AIME, LiveCodeBench, and GPQA-diamond, all of which are publicly available benchmarks. The seed attack prompts are from public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements file, Dockerfile, or detailed environment specification is provided. The paper only mentions using QWen2.5 instruction-tuned models (3B and 7B) without library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. There is no README or 'Reproducing Results' section; the paper describes the method conceptually but does not give runnable instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 1 and Figures 2–5 are presented as point estimates (ASR%, CR%, accuracy) with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'AdvEvo-MARL consistently achieves the lowest ASR') but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Effect sizes as formal measures (Cohen's d, odds ratios) are not reported. Percentage differences are presented in context (e.g., '12% reduction in ASR under NetSafe') but without baseline context sufficient for formal effect size interpretation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of 4,000 training samples, 300 adversarial prompts, and benchmark sizes is not justified. No power analysis or reasoning for the chosen N is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be single-run numbers. No standard deviation, variance across seeds, or multiple-run results with spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against Vanilla (undefended) 3B and 7B models, Challenger (self-verification), Inspector (external guard agent), and closed-source GPT-3.5 and GPT-4o-mini reference models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (Challenger from Huang et al. 2025, Inspector, and the vanilla QWen2.5 models) are contemporary methods from 2024-2025, appropriate for the 2025 submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 compares dynamic vs. static attacker, and individual vs. joint (MAS) defender training. Section 5.4 ablates the public baseline mechanism against a no-baseline variant, with figures showing training dynamics."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses three metrics: Attack Success Rate (ASR), Contagion Rate (CR), and task accuracy/Pass@1, measuring both safety and utility dimensions."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated safety and task performance; human evaluation of system outputs is not relevant to the claims being made about ASR and task accuracy."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses task benchmarks (AIME, GPQA-diamond, LiveCodeBench) and adversarial evaluation sets that are distinct from the training data (MATH-500 for defenders; JailbreakBench/Wild Jailbreak/Strong Reject for evaluation)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-attack-scenario (NetSafe, AutoInject, UserHijack) and per-topology (chain, tree, complete) breakdowns, giving a detailed view of performance across conditions."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that in the complete graph topology, CR remains elevated (up to 35%), and acknowledges that 'even our approach' is not perfectly effective in densely connected environments. Section 5.2 notes that the 3B variant shows a maximum 3% accuracy drop."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation in Section 5.4 shows the no-baseline variant exhibits 'non-stationary behavior and even degraded performance in later stages' including a 13.3% drop in defender response length, constituting a reported negative result for the alternative configuration."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims ASR below 20% and baselines reaching up to 38.33%. Table 1 shows AdvEvo-MARL achieving low ASR across conditions and baselines (e.g., Challenger-7B at 38.33% ASR in UserHijack complete topology), supporting the abstract claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims via ablation studies (dynamic vs. static attacker, with vs. without public baseline, individual vs. joint training) with single-variable manipulation. These ablations provide adequate support for the causal attribution claims in Sections 5.3 and 5.4."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract and conclusion claim AdvEvo-MARL as a 'standardized framework for building MAS that are both safe and capable' broadly, but experiments are limited to QWen2.5 3B/7B models and three specific attack scenarios. No explicit bounding of claims to this setting is made."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed results. For example, the performance improvement of the 7B model on out-of-distribution tasks is presented without considering that co-evolutionary training might improve general RL training dynamics rather than specifically safety-related benefits."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'QWen2.5 instruction-tuned models (3B and 7B)' without specifying the exact model checkpoint, version tag (e.g., 'Qwen2.5-7B-Instruct'), or any snapshot identifier. GPT-3.5 and GPT-4o-mini comparison models also lack version/snapshot dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using adversarial prompts and defender system prompts, but no actual prompt text is provided. The Challenger strategy ('verify the benignness of its input') is described only in natural language without the actual prompt used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The reward weights (αs=1, βt=0.5 in first half, reversed in second half) and the clipping parameter ε and KL coefficient β in Equation 5 are referenced but never assigned specific values. No learning rate, batch size, or other RL training hyperparameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the multi-agent scaffold in detail: three agent topologies (chain, tree, complete), the attacker warm-up via SFT, the adversarial RL loop, the REINFORCE++ training with public baseline, and the reward mechanisms for attackers vs. defenders (Section 4 and Figure 1)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the preprocessing pipeline: sampling 1,000 harmful behaviors, applying jailbreak strategies to get Dinit, using a reasoning model to synthesize traces, filtering with LLM-as-judge to remove invalid trajectories, resulting in ~4,000 samples in Dadv. The filtering criteria (contradictory, off-topic, or vague) are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has an Ethics Statement (Section 7) but it does not function as a limitations section. The conclusion does not include substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. There is no consideration of specific threats such as the small model sizes tested, limited attack scenarios, lack of multiple runs, or potential overfitting to the specific attack strategies used in evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. Claims in the conclusion are broad ('promising and unified framework for building safe and capable multi-agent systems') without bounding to the tested conditions (specific models, attack types, and topologies)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data, model outputs, or evaluation results in a downloadable format are provided. Only aggregated metrics in tables and figures are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes how the training data was collected: sampling 1,000 harmful behaviors from 'existing public datasets,' applying jailbreak strategies, and constructing the evaluation pools from JailbreakBench, Wild Jailbreak, and Strong Reject (Section 5.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper has no human participants; all data comes from automated benchmarks and synthetic adversarial prompt construction."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline from seed prompts to final training data is described: sample harmful behaviors → apply jailbreak strategies → synthesize reasoning traces → filter with LLM judge → obtain ~4,000 samples. Steps and counts are provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section or funding disclosure anywhere in the paper. No grants, corporate sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the first page: Northwestern University, University of Illinois at Chicago, University of Rochester, and Carnegie Mellon University. The paper does not evaluate any product affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement, patent disclosure, or declaration of financial interests in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses QWen2.5 instruction-tuned models but does not state their training data cutoff dates. Evaluations are also conducted against GPT-3.5 and GPT-4o-mini without stating their training cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses AIME, GPQA-diamond, and LiveCodeBench for evaluation but does not discuss whether QWen2.5's pre-training data may have included these benchmarks or their solutions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LiveCodeBench is noted as 'contamination free' (citing Jain et al., 2024) but GPQA-diamond and AIME are public benchmarks that could have appeared in QWen2.5's training data. No contamination analysis is provided for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; an ethics statement is included but addresses the social implications of the research, not IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper proposes a method requiring training multiple LLMs with RL but reports no inference cost, API cost, tokens consumed, or wall-clock time for either training or evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total compute, hardware specifications, or training time are stated anywhere in the paper."
       }
     }
   },
   "claims": [
     {
       "claim": "AdvEvo-MARL consistently keeps attack-success rate (ASR) below 20%, whereas baselines reach up to 38.33%.",
       "evidence": "Table 1 shows AdvEvo-MARL-7B achieving near-zero to low single-digit ASR in chain and tree topologies. In the complete graph topology, the maximum observed ASR is 17.68% (3B model, UserHijack). Challenger-7B reaches 38.33% ASR in UserHijack complete topology.",
       "supported": "strong"
     },
     {
       "claim": "AdvEvo-MARL preserves and sometimes improves task accuracy (up to +3.67% on reasoning tasks) while enhancing safety.",
       "evidence": "Figure 2 shows AdvEvo-MARL-7B outperforming Vanilla-7B on AIME, GPQA, and LiveCodeBench. The abstract states '+3.67%' but the figure shows approximate values; the main results section (Section 5.2) states 'maximum 3% accuracy drop' for 3B variants and improvements for 7B.",
       "supported": "moderate"
     },
     {
       "claim": "Training with a dynamic co-evolutionary attacker yields superior defender robustness compared to a static attacker baseline.",
       "evidence": "Section 5.3 and Figure 3 show a 12% reduction in ASR under NetSafe and consistently lower ASR across AutoInject and UserHijack when using the dynamic MARL attacker vs. a static prompt pool.",
       "supported": "moderate"
     },
     {
       "claim": "The public baseline mechanism for advantage estimation leads to more stable training and better task and safety outcomes than training without it.",
       "evidence": "Section 5.4 and Figure 5 compare public baseline vs. no-baseline configurations, showing steadily improving accuracy and controlled ASR with baseline, and non-stationary behavior plus a 13.3% response length drop without it.",
       "supported": "moderate"
     },
     {
       "claim": "Training defenders jointly in a MAS setting yields better safety and task utility than training them individually.",
       "evidence": "Section 5.3 states 'AdvEvo-MARL exhibits the highest system safety and task utility across all evaluated settings' vs. individual training (referenced in Figure 3), with 12% robustness gain and 4% task utility enhancement under NetSafe.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AdvEvo-MARL co-evolves attacker and defender agents via MARL to internalize safety in LLM-based multi-agent systems, achieving attack success rates below 20% across three attack scenarios and three system topologies while preserving or slightly improving task performance on math, coding, and reasoning benchmarks. The paper demonstrates that a public baseline mechanism (sharing group-level mean returns) stabilizes training compared to standard advantage estimation. The 7B variant of AdvEvo-MARL outperforms its vanilla counterpart on out-of-distribution task benchmarks, suggesting that adversarial co-evolution provides general capability benefits. Results are compared against self-verification (Challenger) and external guard agent (Inspector) baselines using QWen2.5 3B/7B backbone models.",
   "red_flags": [
     {
       "flag": "No statistical testing or uncertainty quantification",
       "detail": "All results are single-run point estimates with no standard deviation, confidence intervals, or error bars. Claims of superiority (e.g., 'consistently achieves the lowest ASR') are made without statistical tests, making it impossible to assess whether differences are meaningful or due to random variation."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Critical RL hyperparameters (learning rate, batch size, number of training steps, KL coefficient β, clip parameter ε, specific reward weights beyond the general priority scheme) are not reported, making reproduction impossible."
     },
     {
       "flag": "Vague model version specification",
       "detail": "The paper uses 'QWen2.5 instruction-tuned models (3B and 7B)' without specifying the exact model checkpoint or version, and GPT-3.5/GPT-4o-mini comparisons lack snapshot dates. This is critical for reproducing and comparing results."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations section or threats-to-validity discussion. Notable unaddressed issues include: small model sizes tested (3B/7B may not represent frontier safety challenges), three specific attack types (generalizability to other attacks is untested), and single-run results with potential high variance."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "AIME, GPQA-diamond, and similar benchmarks are used for task evaluation with QWen2.5 models whose training data cutoffs are not stated. The possibility that these benchmarks appeared in pre-training data is not discussed for most benchmarks."
     },
     {
       "flag": "Overly broad generalization claims",
       "detail": "The conclusion describes AdvEvo-MARL as 'a promising and unified framework for building safe and capable multi-agent systems' broadly, but experiments only cover two model sizes (3B/7B), three attack types, and one model family (QWen2.5). The framework has not been validated on larger models or different LLM families."
     },
     {
       "flag": "Contagion rate anomaly for AdvEvo-MARL-3B in chain topology",
       "detail": "In Table 1, AdvEvo-MARL-3B shows only 6.93% ASR but 35.64% CR under NetSafe in chain topology, which is comparable to the Vanilla-3B model's 36.14% CR despite much lower ASR. This inconsistency is not explained or discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "NetSafe: Exploring the Topological Safety of Multi-Agent Networks",
       "authors": ["Miao Yu", "Shilong Wang", "Guibin Zhang", "Junyuan Mao", "Chenlong Yin", "Qijiong Liu", "Qingsong Wen", "Kun Wang", "Yang Wang"],
       "year": 2024,
       "arxiv_id": "2410.15686",
       "relevance": "Introduces the NetSafe attack framework used as one of the primary evaluation scenarios and a baseline comparison in this paper."
     },
     {
       "title": "On the Resilience of LLM-based Multi-Agent Collaboration with Faulty Agents",
       "authors": ["Jen tse Huang", "Jiaxu Zhou", "Tailin Jin", "Xuhui Zhou", "Zixi Chen", "Wenxuan Wang", "Youliang Yuan", "Michael R. Lyu", "Maarten Sap"],
       "year": 2025,
       "arxiv_id": "2408.00989",
       "relevance": "Provides the AutoInject attack strategy used in this paper's evaluation and proposes self-verification and guard agent baselines."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han", "Alex Gu", "Wen-Ding Li", "Fanjia Yan", "Tianjun Zhang", "Sida Wang", "Armando Solar-Lezama", "Koushik Sen", "Ion Stoica"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Task benchmark used to evaluate coding capabilities; relevant as a contamination-aware coding evaluation benchmark for LLMs."
     },
     {
       "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland", "Jackson Petty", "Richard Yuanzhe Pang", "Julien Dirani", "Julian Michael", "Samuel R. Bowman"],
       "year": 2024,
       "relevance": "General reasoning benchmark used to evaluate task utility of the safety-trained agents; widely used for LLM capability evaluation."
     },
     {
       "title": "REINFORCE++: A Simple and Efficient Approach for Aligning Large Language Models",
       "authors": ["Jian Hu"],
       "year": 2025,
       "arxiv_id": "2501.03262",
       "relevance": "The RL training algorithm used in AdvEvo-MARL for policy optimization, directly relevant to LLM alignment via RL."
     },
     {
       "title": "Evo-MARL: Co-Evolutionary Multi-Agent Reinforcement Learning for Internalized Safety",
       "authors": ["Zhenyu Pan", "Yiting Zhang", "Yutong Zhang", "Jianshu Zhang", "Haozheng Luo", "Yuwei Han", "Dennis Wu", "Hong-Yu Chen", "Philip S. Yu", "Manling Li", "Han Liu"],
       "year": 2025,
       "arxiv_id": "2508.03864",
       "relevance": "A companion/predecessor paper from the same group that appears to be a related earlier version or concurrent work on MARL for safety in multi-agent systems."
     },
     {
       "title": "GuardAgent: Safeguard LLM Agents via Knowledge-Enabled Reasoning",
       "authors": ["Zhen Xiang", "Linzhi Zheng", "Yanjie Li", "Junyuan Hong", "Qinbin Li", "Han Xie", "Jiawei Zhang", "Zidi Xiong", "Chulin Xie", "Nathaniel D Bastian"],
       "year": 2025,
       "relevance": "External guard agent approach for LLM agent safety, representing the class of defenses that AdvEvo-MARL aims to supersede."
     },
     {
       "title": "AgentPoison: Red-Teaming LLM Agents via Poisoning Memory or Knowledge Bases",
       "authors": ["Zhaorun Chen", "Zhen Xiang", "Chaowei Xiao", "Dawn Song", "Bo Li"],
       "year": 2024,
       "relevance": "Adversarial attack on LLM agents relevant to the safety evaluation context of multi-agent systems."
     },
     {
       "title": "Red-Teaming LLM Multi-Agent Systems via Communication Attacks",
       "authors": ["Pengfei He", "Yupin Lin", "Shen Dong", "Han Xu", "Yue Xing", "Hui Liu"],
       "year": 2025,
       "arxiv_id": "2502.14847",
       "relevance": "Attack methodology on multi-agent communication relevant to the adversarial threat model studied in this paper."
     },
     {
       "title": "G-Safeguard: A Topology-Guided Security Lens and Treatment on LLM-based Multi-Agent Systems",
       "authors": ["Shilong Wang", "Guibin Zhang", "Miao Yu", "Guancheng Wan", "Fanci Meng", "Chongye Guo", "Kun Wang", "Yang Wang"],
       "year": 2025,
       "arxiv_id": "2502.11127",
       "relevance": "Graph-neural-network-based defense for multi-agent system safety, an alternative approach to the RL-based method in this paper."
     },
     {
       "title": "LlamaFirewall: An Open Source Guardrail System for Building Secure AI Agents",
       "authors": ["Sahana Chennabasappa", "Cyrus Nikolaidis", "Daniel Song", "David Molnar", "Stephanie Ding", "Shengye Wan", "Spencer Whitman", "Lauren Deason", "Nicholas Doucette", "Abraham Montilla"],
       "year": 2025,
       "arxiv_id": "2505.03574",
       "relevance": "External guardrail system for AI agent safety, representing the external guard paradigm that AdvEvo-MARL contrasts with."
     }
   ]
 }