ai-research-survey

scan.json (25517B)

---

 {
   "paper": {
     "title": "Among Us: Measuring and Mitigating Malicious Contributions in Model Collaboration Systems",
     "authors": ["Ziyuan Yang", "Wenxuan Ding", "Shangbin Feng", "Yulia Tsvetkov"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.05176"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Our code is available at https://github.com/Ziyuan-Yang/AmongUs' and provides the URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (GSM8k, HumanEval, MMLU-redux, TruthfulQA, DS-1000, IFEval, IFBench, CocoNot, SafetyBench, NLGraph). The adversarial datasets are constructed from existing public benchmarks and the code repository is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'A100 GPUs with 40G VRAM' and specific model names but does not provide a requirements.txt, Dockerfile, or detailed library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a GitHub repository is referenced and experimental details are given in the appendix (hyperparameters, LoRA settings, training epochs), there are no explicit step-by-step reproduction instructions or README with commands described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1 and 2 are reported as point estimates only (e.g., .568, .799). No confidence intervals, error bars, or ± notation are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 notes 'statistically significant performance drops further in underline', indicating significance tests were performed. The paper distinguishes between general drops (gray) and statistically significant drops (underlined)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage performance drops with baselines for context: e.g., 'lowering worst-case performance by 34.9% on average', '7.12% and 7.94% drop' for reasoning and safety, 'recovering 95.24% of the original performance' (from 0.368 to 0.541). These provide magnitude with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the choice of 10 datasets, 5 domains, or the specific test set sizes (ranging from 140 to 1000 examples as shown in Table 4). No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures across experimental runs are reported. Results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each collaboration method is evaluated with a benign-only baseline (no malicious models) against which malicious settings are compared, shown as the first row of each section in Tables 1 and 2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The collaboration methods studied include recent work: RouteLLM (Ong et al., 2025), GraphRouter (Feng et al., 2025e), and the MoCo framework (Feng et al., 2026). The model pool uses Qwen2.5-7B-Instruct (2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically ablates across four malicious methods (M1-M4), studies the effect of malicious model count (Section 5, Figure 3), maliciousness diversity (Figure 2), steering factor alpha (Table 6), and out-of-domain SFT transfer (Figure 4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Performance is evaluated across 10 datasets spanning 5 domains (safety, reasoning, knowledge, coding, instruction following), each with domain-appropriate metrics (accuracy, pass rates, compliance scores)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper relies entirely on automated evaluation (regex matching, LLM-based verifier, multiple-choice accuracy, sandbox execution). The Limitations section acknowledges 'Incorporating human evaluation and real-world task settings is an important direction for future work.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 shows explicit dev/test splits for all 10 datasets. The routers are trained on development sets and results are reported on separate test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-dataset breakdowns across all 10 benchmarks, and Table 2 shows per-benchmark mitigation results. The analysis discusses domain-specific impacts (safety vs. reasoning vs. coding)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 7-15 in the appendix provide qualitative examples of both correct and incorrect (malicious) responses. Section 4.2 discusses domains where mitigation fails: 'the average recovery rate still remains around 89.2% for the CocoNot benchmark, and the worst case drops to approximately 75%.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that prompting-based attacks have 'relatively limited impact' (3.24% and 3.65%), some malicious settings showed no degradation (LOGIT AVERAGE, GREEDY SOUP), and mitigation strategies fail to fully recover safety performance. It also notes that 'fully mitigating the impact of malicious models in safety-critical collaboration remains an open problem.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'performance is lowered by 7.12% and 7.94% on average' for reasoning and safety, and mitigation 'recover 95.31% of the initial performance.' These specific numbers are supported by Tables 1 and 2 in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about malicious models degrading performance. These are justified through controlled experiments: the only variable changed is the introduction of malicious models into a fixed collaboration system, with benign-only baselines. The ablation design (varying malicious method, count, domain) constitutes controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly bounds findings: 'our experiments focus on a limited set of collaboration methods... and model architectures', 'the conclusions may not directly generalize to all emerging collaboration paradigms or proprietary systems', and 'mitigation strategies are evaluated primarily on API-level and Text-level collaboration methods.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for the observed performance degradation. For example, it does not consider whether the degradation could be partly attributed to adding any additional (non-malicious but lower-quality) model to the pool, or whether the router/collaboration methods simply fail with any heterogeneous model addition."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'Qwen2.5-7B-Instruct', 'Llama-3.1-8B-Instruct', 'Olmo-3-7B-Instruct', 'Llama-3.1-Tulu-3-8B-DPO', 'Mistral-7B-Instruct-v0.2', 'Skywork-Reward-Llama-3.1-8B', and 'sentence-transformers/all-MiniLM-L6-v2'. These include version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full malicious model prompt for M1-Prompting is provided in Appendix A.1. The paper states TEXT DEBATE 'employs prompts in (Du et al., 2024)' referencing the original work. The malicious persona prompt is given verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 specifies 'top-p = 0.9, temperature=0.7 and 256 maximum generation length.' LoRA training uses 'rank 16, batch size 32... 5 epochs'. GRAPH ROUTER: '500 epochs and 32 batch size.' Logit contrastive: 'λ = 0.2'. Activation steering: 'α = 5.0'."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. The collaboration methods (routing, debate, logit fusion, model merging) are model collaboration approaches, not agentic scaffolding with tools, memory, or retry logic."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes how adversarial datasets were constructed for each domain (M3-SFT), including source datasets, sizes, and how GPT-4O was used to generate misaligned outputs. Appendix A.3 describes model pool construction with specific training data subsets. Dataset dev/test splits are documented in Table 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present, discussing four specific limitations including the scope of malicious behaviors, limited collaboration methods, mitigation only for API/text-level, and reliance on automated evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'Our malicious settings may not fully capture more sophisticated or adaptive adversaries', 'the conclusions may not directly generalize to all emerging collaboration paradigms or proprietary systems', and 'mitigation strategies are evaluated primarily on API-level and Text-level collaboration methods.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section states specific boundaries: mitigation not tested for logit-level and weight-level, limited to studied collaboration methods and model architectures, and acknowledges results use automated evaluation rather than real-world deployment scenarios."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the code repository is provided, the paper does not mention releasing raw experimental outputs, model predictions, or intermediate results that would enable independent verification of the reported numbers."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes adversarial dataset construction in detail: sources, sizes (e.g., '6k vulnerable coding dataset from (Betley et al., 2025)', '7.5k misaligned MATH questions'), and methods. Appendix A.4 describes benchmark evaluation procedures."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; all data comes from publicly available benchmarks and model-generated outputs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: malicious model construction (Section 2.1, Appendix A.1) → model pool integration (Section 2.2) → collaboration execution → evaluation via automated judges. Training details, dataset sizes, and evaluation methods are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section listing funding sources is present in the paper. No mention of grants or corporate sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Washington (Yang, Feng, Tsvetkov) and New York University (Ding). The authors are academic researchers, not employees of the model providers being studied."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding disclosure means this criterion cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-7B-Instruct and evaluates on benchmarks like HumanEval and GSM8k but does not state the training data cutoff date for any of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the test benchmarks (HumanEval, GSM8k, MMLU-redux, etc.) may have appeared in the training data of Qwen2.5-7B-Instruct or other models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), GSM8k (2021), and other benchmarks predate the models used. No discussion of contamination risk for these well-known benchmarks that are widely present in training corpora."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The Ethics Statement discusses responsible use of adversarial research but no IRB is needed."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, tokens consumed, or wall-clock time are reported. The paper mentions A100 GPUs but does not quantify the cost per experiment or total compute."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only 'A100 GPUs with 40G VRAM' is mentioned. No total GPU hours, training time, or computational budget is stated for the extensive experiments (8 collaboration methods x 4 malicious settings x 10 datasets)."
       }
     }
   },
   "claims": [
     {
       "claim": "Malicious models have a severe impact on multi-LLM collaboration systems, lowering worst-case performance by 34.9% on average.",
       "evidence": "Table 1 shows GRAPH ROUTER with M2-Steering has the largest macro-average degradation of 34.99%. Section 4.1 reports this finding.",
       "supported": "strong"
     },
     {
       "claim": "Safety and reasoning domains are disproportionately affected, with 7.94% and 7.12% average performance drops respectively.",
       "evidence": "Table 1 results show CocoNot (safety) average drop of 22.05% and GSM8k (reasoning) average drop of 11.15% across collaboration methods (Section 4.1).",
       "supported": "strong"
     },
     {
       "claim": "Mitigation strategies recover 95.31% of initial performance on average.",
       "evidence": "Table 2 shows recovery rates across benchmarks and collaboration methods with supervisor-based methods. Section 4.2 reports the average recovery rate.",
       "supported": "strong"
     },
     {
       "claim": "Deeper collaborations (logit-level and weight-level) are more robust to malicious models.",
       "evidence": "Table 1 shows LOGIT AVERAGE and GREEDY SOUP have minimal macro-average degradation compared to API-level and text-level methods. Section 4.1 discusses this.",
       "supported": "strong"
     },
     {
       "claim": "Activation steering and RL produce more impactful malicious models than prompting or SFT.",
       "evidence": "Section 4.1 reports M2-steering causes the most impact on API-level methods, and M4-RL causes 12.29% average drop, versus prompting at 3.24% and 3.65%. Table 1 supports these numbers.",
       "supported": "strong"
     },
     {
       "claim": "Concentrated/uniform malicious behaviors are more detrimental than diverse malicious patterns.",
       "evidence": "Figure 2 and Section 5 (Maliciousness Diversity) show performance degrades as malicious diversity decreases, described as a 'counterintuitive result.'",
       "supported": "moderate"
     },
     {
       "claim": "Supervisor-based mitigation outperforms supervisor-free mitigation by 4.5% recovery rate on average.",
       "evidence": "Section 4.2 reports this finding based on Table 2 results across M2 and M4 settings and five benchmarks.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Malicious models injected into multi-LLM collaboration systems (routing, debate, logit fusion, model merging) cause significant performance degradation, with worst-case drops of 34.9% particularly affecting safety and reasoning domains. Deeper collaboration methods (logit-level, weight-level) are more robust than API-level and text-level approaches. Proposed mitigation strategies using external supervisors (LLM-as-a-judge or reward models) recover 95.31% of initial performance on average, though safety-critical tasks remain challenging with recovery rates around 89.2%. Cross-domain malicious fine-tuning can transfer negative effects but is generally weaker than in-domain attacks.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "All results across Tables 1 and 2 are single-point estimates with no standard deviations, confidence intervals, or error bars. With stochastic model generation (top-p=0.9, temperature=0.7), run-to-run variance could be substantial."
     },
     {
       "flag": "Benchmark contamination not discussed",
       "detail": "The paper uses well-known benchmarks (HumanEval, GSM8k, MMLU) with models that may have been trained on them. While contamination is less critical for measuring relative degradation, baseline performance levels may be inflated."
     },
     {
       "flag": "No compute budget disclosed",
       "detail": "The paper runs 8 collaboration methods x 4 malicious settings x 10 datasets plus mitigation experiments and analyses, but provides no total compute cost or GPU hours, making it difficult to assess practical replicability."
     },
     {
       "flag": "Missing alternative explanation for degradation",
       "detail": "The paper does not control for whether adding any additional model (even a non-malicious but differently-skilled one) to the pool would cause similar degradation. The comparison is always benign-pool vs. benign-pool + malicious, never benign-pool vs. benign-pool + random additional benign model."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent misalignment: Narrow finetuning can produce broadly misaligned LLMs",
       "authors": ["Jan Betley", "Daniel Chee Hian Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans"],
       "year": 2025,
       "relevance": "Studies how adversarial fine-tuning in one domain can trigger universal maliciousness that transfers across domains, directly motivating the cross-domain SFT analysis in this paper."
     },
     {
       "title": "RouteLLM: Learning to route LLMs from preference data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang"],
       "year": 2025,
       "relevance": "Proposes the LLM Router approach for routing queries to appropriate models, one of the key collaboration methods evaluated in this paper."
     },
     {
       "title": "GraphRouter: A graph-based router for LLM selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Proposes graph neural network-based routing for LLM selection, the other API-level collaboration method evaluated."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B. Tenenbaum", "Igor Mordatch"],
       "year": 2024,
       "relevance": "Introduces multi-agent debate for improving LLM factuality, one of the text-level collaboration methods evaluated under malicious settings."
     },
     {
       "title": "On the resilience of multi-agent systems with malicious agents",
       "authors": ["Jen tse Huang", "Jiaxu Zhou", "Tailin Jin"],
       "year": 2025,
       "relevance": "Studies how malicious agents affect multi-agent system behavior, directly related prior work on adversarial multi-LLM safety."
     },
     {
       "title": "Adaptive attacks break defenses against indirect prompt injection attacks on LLM agents",
       "authors": ["Qiusi Zhan", "Richard Fang", "Henil Shalin Panchal", "Daniel Kang"],
       "year": 2025,
       "relevance": "Studies adaptive attacks against LLM agent defenses, relevant to understanding attack-defense dynamics in multi-agent systems."
     },
     {
       "title": "G-safeguard: A topology-guided security lens and treatment on LLM-based multi-agent systems",
       "authors": ["Shilong Wang", "Guibin Zhang", "Miao Yu"],
       "year": 2025,
       "relevance": "Proposes topology-guided security methods for multi-agent LLM systems, a mitigation approach for the same threat model studied here."
     },
     {
       "title": "Moco: A one-stop shop for model collaboration research",
       "authors": ["Shangbin Feng", "Yuyang Bai", "Ziyuan Yang"],
       "year": 2026,
       "relevance": "Framework used for implementing all collaboration methods in this paper, representing the state-of-the-art infrastructure for model collaboration research."
     },
     {
       "title": "Watch out for your agents! Investigating backdoor threats to LLM-based agents",
       "authors": ["Wenkai Yang", "Xiaohan Bi", "Yankai Lin"],
       "year": 2024,
       "relevance": "Studies backdoor threats to LLM-based agents, a complementary attack vector to the malicious model injection studied here."
     },
     {
       "title": "Peacemaker or troublemaker: How sycophancy shapes multi-agent debate",
       "authors": ["Binwei Yao", "Chao Shang", "Wanyu Du"],
       "year": 2025,
       "relevance": "Studies how sycophantic behavior affects multi-agent debate quality, related to understanding failure modes in LLM collaboration."
     },
     {
       "title": "When one LLM drools, multi-LLM collaboration rules",
       "authors": ["Shangbin Feng", "Wenxuan Ding", "Alisa Liu"],
       "year": 2025,
       "relevance": "Foundational survey on multi-LLM collaboration paradigms that motivates and contextualizes this security-focused study."
     },
     {
       "title": "NetSafe: Exploring the topological safety of multi-agent system",
       "authors": ["Miao Yu", "Shilong Wang", "Guibin Zhang"],
       "year": 2025,
       "relevance": "Explores topological safety properties of multi-agent systems, relevant to understanding structural vulnerabilities in collaborative AI."
     }
   ]
 }