ai-research-survey

scan-v5.json (25408B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Learning Decentralized LLM Collaboration with Multi-Agent Actor Critic",
     "authors": [
       "Shuo Liu",
       "Tianle Chen",
       "Ryan Amiri",
       "Christopher Amato"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.21972",
     "doi": "10.48550/arXiv.2601.21972"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about MAGRPO/CoLLM-DC matching CoLLM-CC in dense-reward settings and underperforming in sparse-reward/long-horizon settings are directly supported by Table 1 and Figure 2 results across all five tasks.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about CoLLM-CC outperforming alternatives are backed by controlled comparisons with matched hyperparameter budgets across 5 runs; theoretical propositions (4.1–4.3) provide mechanistic justification.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion broadly claims 'MARL-based methods can achieve equal or better performance than a single larger model' from proof-of-concept experiments with 2–100 test examples per task using 1.7B–4B models; limitations acknowledge this but the main text overstates generalizability.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "CoLLM-CC's advantages are attributed solely to variance reduction and critic stationarity; alternative explanations such as critic model capacity, specific reward shaping choices, or task-specific artifacts are not considered.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates high reward with 'high-quality content' (Section 6.3) but writing quality is measured via automated proxies (Jaccard similarity, transition word frequency, length ratios) without validation that these proxies correlate with actual human-judged quality.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix I contains a dedicated Limitations and Future Work section discussing proof-of-concept scale, compute constraints, CoLLM-DC parameter-sharing trade-offs, and open questions about scaling.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Limitations reference 'proof-of-concept settings' and compute constraints broadly but do not specifically address the 2-example Minecraft test sets, the validity of automated writing proxies, absence of significance tests, or benchmark contamination.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that scaling to larger, more heterogeneous multi-agent systems remains open, and that the strictly decentralized no-communication assumption bounds the applicable settings.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "NSF grants (#2044993, #2409351, and multiple others) and computing grants via NCAR and Lambda's Research Grant Program are disclosed in the Acknowledgment section.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are affiliated with Northeastern University, Boston, MA, disclosed on the title page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent government funder with no commercial stake in the MARL framework being evaluated.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests, patent, or financial interests statement appears anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Decentralized LLM collaboration (Section 3.1), LLM Dec-POMDP (Section 3.2), and actor-critic methods including DC and CC variants (Section 4.2) are formally defined with mathematical notation.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions are explicitly enumerated in the introduction: MAAC methods for decentralized LLM collaboration, two concrete algorithms (CoLLM-CC and CoLLM-DC), and empirical validation across three domains.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 situates the work relative to LLM collaboration frameworks with predefined protocols and MARL actor-critic literature, explaining how this work extends beyond both.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code released at https://github.com/OpenMLRL/CoMLRL/releases/tag/v1.3.6 and three task-specific repositories (writing, coding, Minecraft) all at version 1.3.6.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Public datasets (TLDR, arXiv public datasets, HumanEval, MBPP) are used, and the novel CoopHE dataset is available through the GitHub repositories listed in Appendix H.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Hardware specs and model names are listed in Appendix G but no requirements.txt, Dockerfile, or pip dependency list is provided in the paper or referenced for the repositories.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "GitHub repositories are linked; detailed hyperparameters (Appendix C.3), model architectures (C.2), dataset splits (C.1), prompts (E), and reward functions (F) provide sufficient detail to reconstruct experiments.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 explicitly displays 95% bootstrapped confidence intervals as shaded regions around all learning curves for all three methods.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, Mann-Whitney, etc.) are reported for performance comparisons in Table 1; results are averaged over 5 runs without p-values.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports absolute performance percentages for all methods, enabling direct comparison of effect magnitudes with baseline context (e.g., CoLLM-CC 75.2% vs MAGRPO 74.3% vs CoLLM-DC 59.1% on CoopHE).",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The choice of 5 runs is not justified by power analysis; critically, Minecraft test sets contain only 2 examples each (StrBuild[8:10], HouseBuild[8:10]) with no justification for this size.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Figure 2 shows 95% bootstrapped confidence intervals for all learning curves; Table 1 results are described as averaged over 5 runs.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three baseline categories are included: single-model (raw + fine-tuned), multi-agent test-time interaction (parallel, pipeline, discussion), and MARL (MAGRPO).",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "MAGRPO (2026), current Qwen3 models, and recent frameworks are used; baselines are drawn from the same model families as the proposed methods.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Comparison between CoLLM-CC (centralized critic) and CoLLM-DC (decentralized critics) is a direct ablation of the key design choice, and MAGRPO vs MAAC ablates the Monte Carlo vs actor-critic distinction.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports response time, token cost, and task-specific performance; additional metrics include pass@k, IoU, adjacency rate, health points, and training overhead (Table 3).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of generated outputs is performed; writing quality is assessed entirely via automated metrics without human judgment validation.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.1 specifies separate training and test set indices for all tasks (e.g., TLDR[0:1000] train vs TLDR[1000:1100] test).",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 provides per-dataset breakdowns across all 5 task variants; Table 2 provides pass@k breakdowns for CoopHE; Figure 2 shows per-task learning curves.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "CoLLM-DC's failure to converge on Minecraft tasks and MAGRPO's sample inefficiency are discussed with mechanistic attribution (non-stationarity accumulation, reward sparsity).",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results reported: CoLLM-DC fails to converge on long-horizon tasks, and prompt-based multi-agent methods underperform single-model baselines without MARL optimization.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Exact model names provided: Qwen3-1.7B, Qwen2.5-Coder-3B, Qwen3-4B-Instruct-2507, Qwen2.5-3B-Instruct, Qwen2.5-7B Coder and Instruct variants.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix E provides complete verbatim prompts for all task types: TLDR summarization, arXiv expansion, coding collaboration, StrBuild, and HouseBuild agents.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.3 provides comprehensive hyperparameters per task: learning rates, temperature, top-p, buffer sizes, epoch counts, advantage clip, and evaluation sample counts.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The MAAC framework is described via formal pseudocode (Algorithms 1 and 2) with detailed rollout, replay buffer, and training phase descriptions for both CC and DC variants.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Dataset splits are documented in Appendix C.1; CoopHE construction from HumanEval/MBPP selection criteria is described in Section 6.1; reward computation pipelines are in Appendix F.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Public datasets (TLDR, arXiv) are accessible via standard sources, and CoopHE is available through GitHub repositories listed in Appendix H.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "CoopHE construction is described: problems requiring cooperative decomposition were selected from HumanEval/MBPP with the auxiliary function named 'aux' and the main function signature provided in the prompt.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all experiments use automated benchmark evaluation on pre-existing or constructed datasets.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from dataset splits (Appendix C.1) through reward computation (Appendix F) to evaluation metrics is documented across the appendices.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for Qwen3 and Qwen2.5 models are not stated; this is directly relevant since CoopHE derives from HumanEval/MBPP which likely appear in pre-training corpora.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential overlap between the pre-trained models' training data and benchmark tasks (HumanEval, MBPP) used in evaluation.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "HumanEval (2021) and MBPP (2021) predate Qwen model training cutoffs, making contamination plausible; this is not addressed or acknowledged in the paper.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 1 reports response time (seconds on RTX 5090) and token cost (tokens/agent/turn) for all methods across all five tasks.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Appendix G lists all hardware used for training and inference; Appendix D.2 provides training overhead in H200 hours, VRAM usage in GB, and sample/update counts for all methods.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "CoLLM-CC consistently outperforms MAGRPO and CoLLM-DC in long-horizon, sparse-reward tasks.",
       "evidence": "Table 1 shows CoLLM-CC achieving 75.2% pass rate on CoopHE vs MAGRPO 74.3% and CoLLM-DC 59.1%; 68.5% IoU on StrBuild vs 50.6% and 44.6%; 52.7% on HouseBuild vs 50.9% and 46.8%.",
       "supported": "strong"
     },
     {
       "claim": "Monte Carlo methods require substantially more samples to converge in sparse-reward and long-horizon settings.",
       "evidence": "Figure 2c shows MAGRPO reaching stability at ~5000 timesteps vs ~2000 for CoLLM-CC on CoopHE; Proposition 4.3 derives that required inference calls grow as nK(K^H-1)/(K-1).",
       "supported": "strong"
     },
     {
       "claim": "CoLLM-DC fails to converge in long-horizon tasks due to non-stationarity from local-only critic conditioning.",
       "evidence": "Figure 2d/2e show CoLLM-DC underperforming substantially on Minecraft tasks; the paper attributes this to non-stationarity accumulating across 4 turns when critics condition only on local history.",
       "supported": "moderate"
     },
     {
       "claim": "In dense-reward, short-horizon settings, all MARL methods achieve comparable performance.",
       "evidence": "Table 1 writing tasks: CoLLM-CC 95.2%/95.0%, CoLLM-DC 95.4%/94.1%, MAGRPO 93.5%/93.1% on TLDR/arXiv respectively.",
       "supported": "strong"
     },
     {
       "claim": "MARL fine-tuning achieves comparable or better task performance than a comparable single larger model.",
       "evidence": "Table 1 shows MARL methods matching/exceeding single-model baselines on writing, but single-model AC achieves highest health points in HouseBuild (55.9% vs CoLLM-CC 52.7%).",
       "supported": "moderate"
     },
     {
       "claim": "Decentralized MARL reduces inference time and token cost compared to single-model approaches.",
       "evidence": "Table 1: MAGRPO achieves 1.8s / 178 tokens/agent vs raw model 5.0s / 465 tokens on TLDR, attributing this to shorter, coordination-optimized responses.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "theoretical"
   ],
   "key_findings": "Multi-Agent Actor-Critic methods with a centralized critic (CoLLM-CC) outperform Monte Carlo-based MARL (MAGRPO) and decentralized critic approaches (CoLLM-DC) in long-horizon, sparse-reward LLM collaboration tasks, while all methods converge comparably in dense-reward, short-horizon settings. Theoretical analysis (Propositions 4.1–4.3) explains these differences: MC methods suffer exponential sample growth with horizon, and decentralized critics accumulate non-stationarity. Decentralized MARL fine-tuning delivers faster inference and lower token costs than single larger models. Code, prompts, datasets, and hyperparameters are fully released.",
   "red_flags": [
     {
       "flag": "Tiny Minecraft test sets",
       "detail": "StrBuild and HouseBuild use only 2 test examples each (indices 8:10), making performance estimates on these tasks highly unreliable despite 5-run averaging — 10 total observations per method."
     },
     {
       "flag": "No significance tests",
       "detail": "Table 1 performance comparisons lack p-values or hypothesis tests; several differences between methods are small (1–3pp) and may not be statistically meaningful."
     },
     {
       "flag": "Proxy writing metrics unvalidated",
       "detail": "Writing quality claims are based on automated proxies (Jaccard similarity, transition word frequency, length ratios) without human evaluation or validation that these correlate with human-judged quality."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "CoopHE derives from HumanEval (2021) and MBPP (2021), which predate Qwen model training cutoffs; no discussion of whether base models have memorized these problems."
     },
     {
       "flag": "Proof-of-concept scale only",
       "detail": "All models are 1.7B–4B parameters and task sets are small (up to 1000 training examples); the main text makes broad performance claims while the limitations section hedges to proof-of-concept."
     }
   ],
   "cited_papers": [
     {
       "title": "LLM Collaboration with Multi-Agent Reinforcement Learning (Liu et al., 2026a)",
       "relevance": "Direct predecessor from same group proposing MAGRPO; this paper extends it with actor-critic methods"
     },
     {
       "title": "Multi-Agent Actor-Critic for Mixed Cooperative-Competitive Environments (Lowe et al., 2017)",
       "relevance": "Foundational MADDPG paper introducing centralized-critic training with decentralized execution"
     },
     {
       "title": "Counterfactual Multi-Agent Policy Gradients (Foerster et al., 2018)",
       "relevance": "Key MARL paper introducing centralized critics for cooperative settings, directly cited for CTDE"
     },
     {
       "title": "On Centralized Critics in Multi-Agent Reinforcement Learning (Lyu et al., 2023)",
       "relevance": "Theoretical analysis of centralized vs decentralized critics that directly informs this work's analysis"
     },
     {
       "title": "The Surprising Effectiveness of PPO in Cooperative Multi-Agent Games (Yu et al., 2022)",
       "relevance": "MAPPO: centralized-critic MARL approach forming part of the theoretical backdrop"
     },
     {
       "title": "DeepSeek-R1: Incentivizes Reasoning in LLMs through Reinforcement Learning (Guo et al., 2025)",
       "relevance": "Key reference for RLVR training paradigm used in this work"
     },
     {
       "title": "ChatDev: Communicative Agents for Software Development (Qian et al., 2024)",
       "relevance": "Representative multi-agent LLM collaboration framework used as comparison baseline"
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Chen et al., 2021)",
       "relevance": "HumanEval benchmark — source material for CoopHE coding collaboration dataset"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released code enables practitioners to apply MAAC to multi-LLM collaboration tasks, though implementation requires ML expertise and proof-of-concept scale limits confidence."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Centralized-critic advantages in MARL are well-established; applying this to LLM fine-tuning confirms expected behavior rather than overturning conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns are raised; the paper focuses purely on performance optimization of collaborative agents."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper positions against centralized execution protocols dominant in LLM collaboration, arguing for decentralized alternatives, but without major controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and datasets are publicly released on GitHub with versioned releases; Minecraft building demos are visually compelling and runnable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Northeastern University is a solid research institution but not a top-tier AI lab; no famous models or products are involved."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "46822095",
         "title": "Addressing Asymptomatic AI Harms for Dignified Human-AI Interaction",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46822095",
         "created_at": "2026-01-30T08:59:56Z"
       }
     ],
     "top_points": 1,
     "total_points": 1,
     "total_comments": 0
   }
 }