ai-research-survey

scan.json (26450B)

---

 {
   "paper": {
     "title": "Learning Decentralized LLM Collaboration with Multi-Agent Actor Critic",
     "authors": ["Shuo Liu", "Tianle Chen", "Ryan Amiri", "Christopher Amato"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.21972",
     "doi": "10.48550/arXiv.2601.21972"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes CoLLM-CC and CoLLM-DC, multi-agent actor-critic methods for decentralized LLM collaboration. CoLLM-CC with a centralized critic consistently outperforms Monte Carlo methods (MAGRPO) and CoLLM-DC, particularly in long-horizon and sparse-reward settings (Minecraft, coding). In short-horizon dense-reward settings (writing), all three MARL methods achieve comparable performance. Decentralized multi-agent MARL methods can match or exceed single larger model performance while enabling parallel inference.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/OpenMLRL/CoMLRL/releases/tag/v1.3.6 with separate repos for each experiment domain (writing, code generation, Minecraft), listed in Appendix H."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets (TLDR, arXiv-public-datasets) and releases CoopHumanEval as a new dataset in the code repositories (Appendix H). Minecraft tasks are described with specifications."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or dependency version list is provided. Hardware is listed (Appendix G) but software environment specifications are absent."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code repositories are linked, the paper does not include step-by-step reproduction instructions, README commands, or scripts to replicate main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows 95% bootstrapped confidence intervals as shaded regions. The paper states 'Shaded regions denote 95% bootstrapped confidence intervals.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims of 'outperforms' and 'comparable' are based on comparing point estimates and visual inspection of confidence intervals, without formal tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides absolute performance numbers with baselines for context (e.g., CoLLM-CC 75.2% pass rate vs MAGRPO 74.3% vs raw model 56.3%), allowing readers to assess magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes. CoopHE has only 82 problems (66 train, 16 test), Minecraft has 10 tasks (8 train, 2 test). No power analysis or justification for these small sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are averaged over 5 runs (Table 1: 'Results are averaged over 5 runs'), and Figure 2 shows 95% bootstrapped confidence intervals capturing run-to-run variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three categories of baselines: single-model (GRPO, AC), multi-agent test-time interaction (parallel, pipeline, discussion), and MARL (MAGRPO). Described in Section 6.2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "MAGRPO is based on Liu et al. 2026a, and test-time interaction baselines follow Wu et al. 2023 and Du et al. 2023. The MARL baselines are contemporary to this work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between CoLLM-CC (centralized critic) and CoLLM-DC (decentralized critics) serves as an ablation of the critic architecture. The paper systematically varies this component across all domains."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics per domain: writing uses structure, style consistency, logical coherence; coding uses pass rate and pass@k; Minecraft uses IoU, adjacency rate, health points. Response time and token cost also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. Writing quality is assessed by automated metrics (length ratio, Jaccard similarity, transition words), not human judges. For a paper about collaborative writing and coding, human evaluation of output quality would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 clearly separates training and test sets for all domains (e.g., TLDR[0:1000] train / TLDR[1000:1100] test, CoopHE[0:66] train / CoopHE[66:82] test)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across 5 tasks (TLDR, arXiv, CoopHE, StrBuild, HouseBuild) in Table 1, with separate learning curves in Figure 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where CoLLM-DC fails to converge (coding tasks at 4500 timesteps, Minecraft tasks), and where MAGRPO struggles (long-horizon settings). Section 6.3 discusses specific failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CoLLM-DC's convergence failures in coding and Minecraft tasks are negative results. The paper also reports that prompt-based multi-agent methods perform poorly in Minecraft, and that single-model fine-tuning yields limited improvement in coding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims CoLLM-CC outperforms in long-horizon/sparse-reward settings while Monte Carlo and CoLLM-DC are comparable in short-horizon/dense-reward settings. Table 1 and Figure 2 support these claims across all domains."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about why CoLLM-CC outperforms (centralized critic provides more accurate value estimates). The controlled experimental design (same agents, same tasks, same learning rate scaling) with ablation of the critic type provides adequate causal evidence for these architecture-level claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Decentralized LLM Collaboration' generally, but experiments use only small models (1.7B-4B parameters) on proof-of-concept tasks. The paper acknowledges this in Limitations (Appendix I) but the title and abstract do not bound the claims to small-scale settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, CoLLM-CC's advantage could stem from having access to more information during training rather than the actor-critic architecture per se. No robustness checks against confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific metrics (pass rate, IoU, adjacency rate, writing scores) and frames claims at the level of those metrics. It does not claim broader 'collaboration quality' beyond what is measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: Qwen3-1.7B, Qwen2.5-Coder-3B, Qwen3-4B-Instruct, Qwen3-4B-Instruct-2507, Qwen2.5-3B-Instruct. These are sufficiently specific versioned model releases."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text for all agents in all three domains is provided in Appendix E, including writing collaboration, coding collaboration, and Minecraft instructions."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides comprehensive hyperparameters: temperature, top-p, max tokens, learning rates, advantage clip, number of generations, rollout buffer size, training epochs, and evaluation samples for all methods and domains."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent framework is described in detail: Algorithm 1 shows the full training procedure, Section 5 describes KV-cache history management, replay buffer, teacher-forced forward passes, and the feedback loop with external tools."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C.1 lists exact dataset splits and indices. The CoopHE construction criteria are described in Section 6.1 (problems that 'naturally admit cooperative decomposition'). Reward design details in Appendix F document how outputs are processed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I is titled 'Limitations and Future Work' and discusses several limitations in substantive detail."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I discusses specific threats: inability to compare with MAGRPO on longer horizons due to sample efficiency, experiments limited to proof-of-concept settings due to compute constraints, and open question of scaling to larger MAS with more diverse agents."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I explicitly states: 'our experiments are limited to proof-of-concept settings. How LLM-based collaboration can scale to larger multi-agent systems with more diverse and heterogeneous agents remains an open question.' Also notes the strictly decentralized no-communication assumption."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (training logs, per-run results) is made available. Only aggregated results in Table 1 and smoothed curves in Figure 2 are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are described: TLDR Reddit posts, arXiv abstracts from arXiv-public-datasets, CoopHE constructed from HumanEval/MBPP filtering, Minecraft tasks with specifications. Collection procedures are clear."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from public datasets and constructed benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from data collection through reward computation is documented: dataset splits (Appendix C.1), reward computation (Appendix F), and training procedure (Algorithm 1). The coding feedback loop including static analysis and sandbox tests is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Acknowledgment) lists NSF grants #2044993 and #2409351, NCSA allocation CIS251326, Northeastern University computing, and Lambda Research Grant Program."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Northeastern University, Boston, MA. No commercial product is being evaluated, so no product-affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSF (federal research grants) and university computing resources. None of these funders have a financial interest in the outcomes of decentralized LLM collaboration research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained Qwen models (Qwen3-1.7B, Qwen2.5-Coder-3B, etc.) but does not state their training data cutoff dates. For CoopHE, it is unclear whether the base models saw HumanEval/MBPP problems during pre-training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Qwen models' pre-training data overlaps with TLDR, arXiv, or HumanEval/MBPP problems used in evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "CoopHE is derived from HumanEval and MBPP, which were published in 2021. The Qwen models used were likely trained after 2021 and may have seen these problems. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports response time (seconds) and token cost (tokens/agent/turn) for all methods across all tasks. Inference device specified as NVIDIA GeForce RTX 5090."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 (Appendix D.2) reports training duration in H200 hours, VRAM usage in GB, total samples, and total updates for each method. Appendix G lists all hardware used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 5 runs with 95% bootstrapped confidence intervals (Figure 2), showing run-to-run variance. Table 1 states 'Results are averaged over 5 runs.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'Results are averaged over 5 runs' (Table 1 caption) and '5 runs' repeated in Table 3."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Appendix C lists final hyperparameters but does not describe how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for how the final hyperparameters were selected. Different learning rates are used for different methods (e.g., MAGRPO 2×10⁻⁵ vs CoLLM 5×10⁻⁶ for coding) without explaining the selection process."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite making many comparisons across 5 tasks and multiple methods."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (MAGRPO, parallel, pipeline, discussion) and do not acknowledge the bias of evaluating against their own implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares training overhead (samples, updates, duration, VRAM) across methods. The paper explicitly discusses the compute-performance tradeoff: MAGRPO needs more samples but less training time."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "CoopHE is a custom benchmark filtering HumanEval/MBPP for 'cooperative decomposition' but the paper does not discuss whether this filtering actually captures meaningful collaborative coding or how well these tasks represent real multi-agent programming."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All MARL methods use the same agent models and prompts, varying only the training algorithm. The scaffold (prompt design, feedback loop) is held constant across MARL comparisons, isolating the critic architecture effect."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) predate the Qwen models. The paper does not discuss whether solutions to these problems appeared in the models' training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the feedback from static analysis and sandbox tests provides information leakage that wouldn't be available in real deployment scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between train and test splits in CoopHE, TLDR, or arXiv datasets. CoopHE problems come from the same source (HumanEval/MBPP)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "CoLLM-CC consistently outperforms Monte Carlo methods and CoLLM-DC, particularly in long-horizon tasks with sparse rewards.",
       "evidence": "Table 1 shows CoLLM-CC achieves 68.5% IoU on StrBuild vs 50.6% for MAGRPO and 44.6% for CoLLM-DC; 86.4% HP and 52.7% IoU on HouseBuild vs 80.2%/50.9% and 43.8%/46.8%. Figure 2d-e show convergence advantages.",
       "supported": "strong"
     },
     {
       "claim": "In short-horizon dense-reward settings, Monte Carlo methods and CoLLM-DC achieve performance comparable to CoLLM-CC.",
       "evidence": "Table 1 shows writing tasks: MAGRPO 93.5/93.1, CoLLM-DC 95.4/94.1, CoLLM-CC 95.2/95.0 on TLDR/arXiv scores. Differences are within confidence intervals in Figure 2a-b.",
       "supported": "strong"
     },
     {
       "claim": "MARL-based methods can achieve equal or better performance than a single larger model while enabling parallel inference.",
       "evidence": "Table 1 shows MARL methods (1.7B-4B agents) match or exceed single 4B-7B model performance on most tasks, with faster inference time (e.g., 1.8s vs 5.0s on TLDR).",
       "supported": "moderate"
     },
     {
       "claim": "K-sampling MA-REINFORCE variance scales as 1/K^(H-t) but inference cost grows as nK(K^H - 1)/(K-1).",
       "evidence": "Propositions 4.2 and 4.3 with formal proofs in Appendix A. The practical implication is demonstrated by setting K=2 for Minecraft (H=4) vs K=4 for writing (H=1).",
       "supported": "strong"
     },
     {
       "claim": "CoLLM-DC fails to converge in long-horizon tasks due to non-stationarity accumulating across turns.",
       "evidence": "Figure 2c-e shows CoLLM-DC oscillating or failing to improve. Section 6.3 attributes this to 'non-stationarity accumulates across 4 turns, leading to more unstable value estimates.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very small evaluation sets",
       "detail": "CoopHE test set has only 16 problems and Minecraft test sets have only 2 tasks each. Results on such small test sets may not be stable or generalizable."
     },
     {
       "flag": "No benchmark contamination analysis",
       "detail": "CoopHE is derived from HumanEval/MBPP (2021). The Qwen models likely trained on data containing these problems. No contamination analysis is performed."
     },
     {
       "flag": "Custom automated writing metrics of questionable validity",
       "detail": "Writing quality is evaluated using Jaccard similarity, transition word frequency, and length ratios. These metrics have no established correlation with actual writing quality. No human evaluation validates the automated metrics."
     },
     {
       "flag": "Unequal hyperparameter tuning across methods",
       "detail": "Different learning rates, buffer sizes, and epoch counts are used for MAGRPO vs CoLLM methods (e.g., MAGRPO 2×10⁻⁵ vs CoLLM 5×10⁻⁶ for coding). No justification for these choices or evidence of fair search budgets."
     }
   ],
   "cited_papers": [
     {
       "title": "ChatDev: Communicative Agents for Software Development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "Multi-agent LLM system for collaborative software development, directly relevant to agentic AI programming workflows."
     },
     {
       "title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Major multi-agent LLM framework enabling collaborative conversation-based problem solving."
     },
     {
       "title": "AgentBench: Evaluating LLMs as Agents",
       "authors": ["Xiao Liu", "Hao Yu", "Hanchen Zhang"],
       "year": 2023,
       "arxiv_id": "2308.03688",
       "relevance": "Benchmark for evaluating LLMs as autonomous agents across diverse environments."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark for code generation, foundational to LLM programming evaluation."
     },
     {
       "title": "DeepSeek-R1 Incentivizes Reasoning in LLMs through Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "He Zhang"],
       "year": 2025,
       "doi": "10.1038/s41586-025-09422-z",
       "relevance": "RL fine-tuning of LLMs for reasoning, directly relevant to RLHF/RLVR methodology for LLM improvement."
     },
     {
       "title": "Multi-Agent Deep Research: Training Multi-Agent Systems with M-GRPO",
       "authors": ["Haotian Hong", "Jia Yin", "Yiwei Wang"],
       "year": 2025,
       "arxiv_id": "2511.13288",
       "relevance": "Multi-agent GRPO method for training collaborative LLM systems, direct comparison baseline."
     },
     {
       "title": "MARFT: Multi-Agent Reinforcement Fine-Tuning",
       "authors": ["Junhao Liao", "Mingzhou Wen", "Jiaqi Wang"],
       "year": 2025,
       "arxiv_id": "2504.16129",
       "relevance": "MARL fine-tuning framework for LLMs, closely related approach to optimizing multi-agent collaboration."
     },
     {
       "title": "Heterogeneous Swarms: Jointly Optimizing Model Roles and Weights for Multi-LLM Systems",
       "authors": ["Shangbin Feng", "Zifeng Wang", "Prithviraj Goyal"],
       "year": 2025,
       "arxiv_id": "2502.04510",
       "relevance": "Optimization of heterogeneous multi-LLM systems with role specialization."
     },
     {
       "title": "The Surprising Effectiveness of PPO in Cooperative Multi-Agent Games",
       "authors": ["Chao Yu", "Akash Velu", "Eugene Vinitsky"],
       "year": 2022,
       "relevance": "Key MARL benchmark paper on cooperative multi-agent PPO, foundational to actor-critic methods used here."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Agent-based automated software engineering system, relevant to agentic AI coding pipelines."
     }
   ]
 }