ai-research-survey

scan.json (19646B)

---

 {
   "paper": {
     "title": "Convergence Dynamics of Agent-to-Agent Interactions with Misaligned Objectives",
     "authors": ["Romain Cosentino", "Sarath Shekkizhar", "Adam Earle"],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2511.08710"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The experiments use synthetic data generated from described distributions, but no code or data archive is released to reproduce them."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using GPT-5-mini but gives no library versions or setup details beyond model name."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While algorithms are described in pseudocode (Algorithms 1 and 2) and hyperparameters are listed in Table 8.2, there are no step-by-step reproduction instructions or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows 'shaded ± std bands' across 100 runs for the adversarial attack experiments."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Comparisons between theory and experiment rely on visual agreement of curves."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes (Cohen's d, relative magnitudes with context) are reported. Results are presented as convergence curves and theoretical bounds."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 100 runs for Figure 4 and 1000 interactions for Figure 2 but does not justify why these numbers are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 reports mean trajectories across 100 runs with shaded ± std bands."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Single-agent convergence is used as a baseline (Figure 3 compares single agent W vs. agent W with cooperative helper U)."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper proposing a novel framework; no prior work exists on the same formulation to compare against."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies objective alignment angle (aligned, orthogonal, opposite in Figure 1 and Figure 4), and contrasts fixed-objective vs. adaptive-objective regimes (Section 3, Corollary 3, Figure 3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only one metric is used: distance to objective (squared error ∥w - w*∥²). No additional metrics are reported."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant for this theoretical/mathematical framework paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper with synthetic experiments validating mathematical predictions, not a benchmark evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by objective alignment condition (aligned ~6°, orthogonal ~90°, opposite ~174° in Figure 1; orthogonal/scaled/opposite in Figure 4) and by agent type (LSA vs GPT-5-mini)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 (Limitations and Scope) explicitly states the framework is restricted to synthetic linear regression and does not directly explain open-ended reasoning. The adversarial regime itself characterizes failure modes of multi-agent systems."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's core finding is partly negative: misaligned agents inevitably plateau at biased fixed points and cannot improve beyond single-agent optima in the fixed-objective regime."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about biased equilibria (Proposition 1, Figure 1), adversarial asymmetric convergence (Proposition 2, Figure 4), and cooperative acceleration (Corollary 3, Figure 3) are all supported by theoretical proofs and experiments."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (misalignment leads to biased equilibria, adversarial geometry enables asymmetric convergence) are justified through formal mathematical proofs with controlled single-variable manipulations in experiments."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly bounds generalization: 'Our experiments are restricted to synthetic in-context linear regression and LSA agents... our results do not directly explain the behavior of multi-agent LLM collaborations on open-ended reasoning, writing, or code-generation benchmarks.' The abstract also says 'within this simplified setting.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 discusses that the LSA model is a proxy and real LLM behavior may differ. Section 7 discusses defense mechanisms. The paper acknowledges the gap between the simplified model and real multi-agent systems."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-5-mini' in Section 5 and Appendix 8.3. While no snapshot date is given, the model name includes a specific variant identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt and response schema for GPT-5-mini are provided in Appendix 8.4, including the exact text sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix 8.2 provides a hyperparameter table (d=10, n=20, batch_size=512, epochs=100, η=0.005, etc.). Appendix 8.3 specifies temperature=0.0, top_p=1.0 for GPT-5-mini."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The agents are simple single-call models performing gradient computation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Appendix 8.1 fully describe data generation: i.i.d. linear regression with X ~ N(0, I/d), w* ~ N(0, I/d), y = X^T w*. Trajectory truncation criterion (gradient norm < 10^-3) is stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 is titled 'Limitations and Scope' and provides substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states specific limitations: experiments restricted to synthetic linear regression, only GPT-5-mini tested on same linear tasks, results don't explain open-ended reasoning/writing/code-generation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly states: 'our results do not directly explain the behavior of multi-agent LLM collaborations on open-ended reasoning, writing, or code-generation benchmarks.' The paper frames itself as 'a mechanistic case study' and 'stepping stone.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data or logs are released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation is fully specified mathematically: synthetic linear regression data from known distributions (Section 2.1, Appendix 8.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data is synthetic."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from data generation to training to inference is documented in Sections 2, 5, and Appendix 8.1-8.4, including trajectory truncation criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed as affiliated with Salesforce AI Research."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Authors are from Salesforce AI Research. No funding disclosure is provided, so independence cannot be assessed. Salesforce builds multi-agent AI products, creating a potential interest in the outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model capability on any benchmark. GPT-5-mini is used to perform gradient computation on synthetic data, not tested on pre-existing benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed; all data is synthetically generated."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-5-mini API calls (100 runs with multiple interaction steps) but does not report API costs, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total computational budget, GPU hours for LSA training, or API spend for GPT-5-mini experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Under fixed misaligned objectives, agent-to-agent dynamics converge to biased fixed points with residual errors governed by objective misalignment and prompt geometry anisotropy.",
       "evidence": "Proposition 1 provides closed-form expressions; Figure 1 validates empirically across aligned (~6°), orthogonal (~90°), and opposite (~174°) settings.",
       "supported": "strong"
     },
     {
       "claim": "Normalized plateau errors are nondecreasing functions of the inter-objective angle θ.",
       "evidence": "Corollary 2 provides formal bounds; Figure 2 validates with 1000 LSA agent-to-agent interactions showing monotonic growth.",
       "supported": "strong"
     },
     {
       "claim": "Asymmetric convergence (one agent reaches objective, other does not) is possible under specific kernel conditions on prompt geometry.",
       "evidence": "Proposition 2 gives necessary and sufficient conditions; Corollary 5 provides constructive recipe; Figure 4 validates with both LSA and GPT-5-mini across 100 runs with std bands.",
       "supported": "strong"
     },
     {
       "claim": "A cooperative helper agent with adaptive turn-based objectives can implement Newton-like acceleration for the main agent.",
       "evidence": "Corollary 3 provides the construction; Figure 3 shows the cooperative agent accelerating convergence beyond single-agent performance in 3 alternating steps.",
       "supported": "strong"
     },
     {
       "claim": "GPT-5-mini experiments are consistent with theoretical predictions from the LSA framework.",
       "evidence": "Figure 4 (top) shows GPT-5-mini adversarial attack results matching theoretical predictions, with early-step variability from model decoding noise.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper develops a theoretical framework for multi-agent interactions modeled as alternating in-context gradient descent between linear self-attention transformers. It proves that misaligned objectives lead to biased convergence plateaus predictable from objective gap and prompt geometry, identifies conditions for adversarial asymmetric convergence where one agent succeeds while biasing the other, and shows that adaptive-objective helper agents can accelerate convergence via Newton-like steps. Experiments with trained LSA agents and GPT-5-mini validate the theoretical predictions.",
   "red_flags": [
     {
       "flag": "Narrow experimental scope",
       "detail": "All experiments are on synthetic in-context linear regression with dimension d=10, n=20. The gap between this simplified setting and real multi-agent LLM interactions (open-ended reasoning, code generation) is substantial. The authors acknowledge this but the paper's title and framing suggest broader applicability."
     },
     {
       "flag": "No code or data release",
       "detail": "Despite being a Salesforce AI Research paper with reproducible synthetic experiments, no code repository is provided, making independent verification difficult."
     },
     {
       "flag": "Company affiliation undisclosed as conflict",
       "detail": "All authors are from Salesforce AI Research, which develops multi-agent AI products (e.g., Agentforce). The paper's findings about agent interactions have direct commercial relevance, but no conflict of interest statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multiagent systems fail?",
       "authors": ["Melissa Z Pan", "Mert Cemri", "Lakshya A Agrawal"],
       "year": 2025,
       "relevance": "MAST taxonomy of multi-agent failure modes, directly connected to the theoretical predictions in this paper."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Major multi-agent LLM framework; the theoretical framework here aims to explain dynamics in such systems."
     },
     {
       "title": "Red-teaming LLM multi-agent systems via communication attacks",
       "authors": ["Pengfei He", "Yuping Lin", "Shen Dong"],
       "year": 2025,
       "relevance": "Empirical work on adversarial attacks in multi-agent LLM systems, complementary to the theoretical adversarial framework here."
     },
     {
       "title": "Rethinking the bounds of LLM reasoning: Are multi-agent discussions the key?",
       "authors": ["Qineng Wang", "Zihao Wang", "Ying Su"],
       "year": 2024,
       "relevance": "Questions whether multi-agent setups consistently outperform single-agent baselines, which this paper formalizes theoretically."
     },
     {
       "title": "Prompt infection: LLM-to-LLM prompt injection within multi-agent systems",
       "authors": ["Donghyun Lee", "Mo Tiwari"],
       "year": 2024,
       "relevance": "Empirical study of prompt injection attacks between LLM agents, related to the adversarial agent dynamics studied here."
     },
     {
       "title": "On the resilience of multi-agent systems with malicious agents",
       "authors": ["Jen-tse Huang", "Jiaxu Zhou", "Tailin Jin"],
       "year": 2024,
       "relevance": "Studies multi-agent system robustness to malicious agents, directly relevant to the adversarial convergence results."
     },
     {
       "title": "Transformers learn in-context by gradient descent",
       "authors": ["Johannes von Oswald", "Eyvind Niklasson", "Ettore Randazzo"],
       "year": 2023,
       "relevance": "Foundational work showing transformers implement gradient descent in-context, which this paper extends to multi-agent settings."
     },
     {
       "title": "Transformers learn to implement multi-step gradient descent with chain of thought",
       "authors": ["Jianhao Huang", "Zixuan Wang", "Jason D Lee"],
       "year": 2025,
       "arxiv_id": "2502.21212",
       "relevance": "Direct theoretical foundation for the LSA agent model used in this paper."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2023,
       "relevance": "Multi-agent debate framework whose dynamics this paper aims to theoretically characterize."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Zhiheng Xi", "Wenxiang Chen", "Xin Guo"],
       "year": 2025,
       "relevance": "Comprehensive survey of LLM-based agents, providing context for the multi-agent interaction dynamics studied here."
     }
   ]
 }