ai-research-survey

scan.json (25933B)

---

 {
   "paper": {
     "title": "Thought Communication in Multiagent Collaboration",
     "authors": ["Yujia Zheng", "Zhuokai Zhao", "Zijian Li", "Yaqi Xie", "Mingze Gao", "Lizhu Zhang", "Kun Zhang"],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2510.20733",
     "doi": "10.48550/arXiv.2510.20733"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper references the baseline code (Multiagent Finetuning GitHub) but does not release its own THOUGHTCOMM implementation."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The experiments use publicly available benchmarks: MATH and GSM8K. The synthetic data is generated from described random processes."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA H100 GPUs' (Appendix C.1) but provides no requirements.txt, library versions, or environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports ± standard deviation for all accuracy scores (e.g., '85.00 ± 1.60')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims THOUGHTCOMM outperforms baselines but provides no statistical significance tests (no p-values, t-tests, or bootstrap tests). Comparisons are based solely on point estimates with standard deviations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute gains and relative improvements, e.g., '17.2% absolute gain over Multiagent Finetuning and a 113.3% relative improvement over the single answer baseline' (§5.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 500 examples for fine-tuning and 500 for evaluation following prior work, but provides no justification for why these sizes are adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for accuracy scores in Table 1 (e.g., '± 2.23')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against single-LLM performance and Multiagent Finetuning (Subramaniam et al., 2025), described as 'current state-of-the-art.'"
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Multiagent Finetuning (2025) is contemporary. The paper acknowledges other multi-agent workflows exist but focuses on validating the paradigm rather than exhaustive comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper varies prefix length (§5.4), number of debate rounds (§5.3), latent dimensionality (Appendix C.3), and number of agents (Appendix C.4), effectively ablating key design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy and consensus score (proportion of unanimous decisions) as metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated (accuracy against ground truths)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 500 examples for fine-tuning and a separate 500 for evaluation (§5.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by base model (5 different LLMs), by benchmark (MATH and GSM8K), and by various hyperparameter settings."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative error analysis or failure cases are discussed. The paper does not examine where THOUGHTCOMM fails or produces incorrect answers."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "On LLaMA 3-8B-Instruct with GSM8K, THOUGHTCOMM (68.40%) slightly underperforms Multiagent Finetuning (69.20%), which is reported honestly in Table 1."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims identifiability guarantees (supported by Theorems 1-3) and collaborative advantages (supported by Table 1 experiments). The claims are generally supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims THOUGHTCOMM 'enables' superior collaboration via 'mind-to-mind communication,' but the ablations do not isolate whether gains come from the latent thought extraction, the prefix injection, or the additional training signal. The design does not control for the additional parameters introduced by the autoencoder and adapter."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the paradigm 'naturally extends beyond LLMs to all modalities' but experiments are limited to math reasoning tasks (MATH, GSM8K) with 5 specific LLMs. The title 'Multiagent Collaboration' is much broader than what is tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the improvements could be due to the additional training (autoencoder + adapter) rather than the thought communication paradigm itself. No confound analysis is provided."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames results as demonstrating 'collaborative advantages of thought communication' and 'mind-to-mind' communication but actually measures math problem accuracy. No discussion of whether math accuracy is a sufficient proxy for the broad collaboration claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: Llama-3-8B-Instruct, Phi-4-mini-instruct, Qwen-3-0.6B, Qwen-3-1.7B, Deepseek-R1-distilled-Llama-8B (§5.2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using debate among agents but does not provide the actual prompts or system instructions used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states prefix token count is 1 (Appendix C.1) but does not report temperature, top-p, learning rate for the autoencoder/adapter, or other critical hyperparameters."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The THOUGHTCOMM framework is described in detail in §4: autoencoder for latent extraction, agreement-based reweighting, and prefix adaptation injection. The architecture is clearly specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the data split (500 train/500 test), selection of harder questions (level-3 complexity in MATH), and evaluation protocol (§5.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Conclusion) discusses limitations: framework requires access to model states (not feasible for closed-source models), and the alternative of context-aware embeddings has not been explored empirically."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifically notes that model states may be inaccessible for closed-source models, which is a concrete limitation specific to this approach."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT show. It does not bound results to math reasoning or specific model sizes. The framing ('beyond LLMs to all modalities') is expansive without clear boundary-setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (individual predictions, model states, latent representations) is released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data comes from established benchmarks (MATH, GSM8K) with clear sampling described (500/500 split, level-3+ for MATH)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data source is standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark data to final evaluation is only partially described. How responses are parsed, how ground-truth matching works, and how the standard deviation is computed are not detailed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section lists NSF Award No. 2229881, NIH Contract R01HL159805, and grants from Quris AI, Florin Court Capital, MBZUAI-WIS Joint Program, and Al Deira Causal Education project."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: CMU, Meta AI, and MBZUAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders (NSF, NIH, academic grants) do not have a direct financial stake in multi-agent communication outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included. Authors from Meta AI could have commercial interests in multi-agent LLM systems, but no declaration is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained LLMs on MATH and GSM8K benchmarks but does not state the training data cutoff dates for any of the models used."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "MATH (2021) and GSM8K (2021) are well-known benchmarks that likely appear in training data for models released in 2024-2025. No discussion of potential overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both MATH and GSM8K were published in 2021, well before the training cutoffs of all evaluated models. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example cost is reported. The paper claims the approach is 'efficient' but provides no quantitative cost data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 8 NVIDIA H100 GPUs but does not state total GPU hours, training time, or compute budget for the autoencoder/adapter training."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Standard deviations are reported but it is unclear whether they come from multiple seeds or are analytical estimates. No explicit seed sensitivity analysis is described."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state how many runs produced the reported results. The ± values in Table 1 appear to be standard errors computed from the binary outcomes rather than multi-run variance."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The latent dimension sweep (§C.3) and prefix length sweep (§5.4) show sensitivity but do not report total search compute."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports results for multiple configurations but does not explain how the default configuration (prefix=1, latent dim unspecified for main results) was selected."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 5 models × 2 benchmarks × multiple settings with no correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors use the baseline's released code but compare their own THOUGHTCOMM implementation against it. No acknowledgment of potential author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "THOUGHTCOMM adds autoencoder + adapter training on top of the base model. Multiagent Finetuning fine-tunes the full LLM. The compute difference is discussed qualitatively but no matched-compute comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "MATH and GSM8K are used to evaluate 'thought communication' and 'collaboration beyond language,' but no discussion of whether math accuracy is a valid measure of these broad claims."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "THOUGHTCOMM introduces a different communication mechanism (latent prefix injection) vs. the baseline (natural language debate). The comparison conflates the scaffold/communication mechanism with the method's value. No attempt to control for the scaffold difference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH and GSM8K (both 2021) predate all evaluated models (2024-2025). No discussion of temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., through debate context or problem format)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 500/500 train/test split is described as random sampling. No discussion of potential non-independence (e.g., similar problem structures across splits)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used or discussed."
       }
     }
   },
   "claims": [
     {
       "claim": "Latent shared and private thoughts between agents can be identified (disentangled) under sparsity regularization in a nonparametric setting.",
       "evidence": "Theorems 1, 2, and 3 in §3 provide formal proofs. Synthetic experiments (§5.1, Figs. 3-4) validate recovery with R² scores and MCC above identifiability threshold.",
       "supported": "strong"
     },
     {
       "claim": "THOUGHTCOMM achieves 67.23% average relative improvement over single-agent and 19.06% over the state-of-the-art Multiagent Finetuning.",
       "evidence": "Table 1 in §5.2 shows results across 5 models and 2 benchmarks. Improvements are consistent across most settings.",
       "supported": "moderate"
     },
     {
       "claim": "THOUGHTCOMM is robust to increasing debate rounds while Multiagent Finetuning degrades.",
       "evidence": "Figure 6 (§5.3) shows accuracy drop for baseline but gains for THOUGHTCOMM from 2 to 6 rounds on LLaMA-3-8B-Instruct. Additional results on Qwen-3-1.7B in Appendix C.2.",
       "supported": "moderate"
     },
     {
       "claim": "THOUGHTCOMM is robust to prefix length, achieving near-optimal performance with a single injected vector.",
       "evidence": "Figure 5 (§5.4) shows <5% fluctuation across prefix lengths 1-16 on four models.",
       "supported": "moderate"
     },
     {
       "claim": "THOUGHTCOMM's training overhead is model-agnostic, depending only on embedding dimension not parameter count.",
       "evidence": "Stated in §5.2. Theoretical argument that autoencoder/adapter depend on embedding dimension. No empirical training cost data provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper introduces THOUGHTCOMM, a framework for multi-agent LLM communication that bypasses natural language by extracting and sharing latent thoughts via a sparsity-regularized autoencoder. Theoretical identifiability guarantees (Theorems 1-3) show shared and private latent thoughts can be recovered. On MATH and GSM8K benchmarks across 5 LLMs, THOUGHTCOMM outperforms Multiagent Finetuning by an average of 19% relative improvement. The approach shows robustness to varying prefix lengths, debate rounds, and number of agents.",
   "red_flags": [
     {
       "flag": "Benchmark contamination risk ignored",
       "detail": "Both MATH (2021) and GSM8K (2021) are well-known benchmarks likely present in the training data of all 5 evaluated models (released 2024-2025). No contamination analysis is performed, which undermines the validity of absolute accuracy numbers. Relative comparisons between methods may still be valid if contamination affects all methods equally."
     },
     {
       "flag": "Confound between additional training and thought communication",
       "detail": "THOUGHTCOMM trains an autoencoder and adapter module, providing additional learning signal. Multiagent Finetuning also trains, but via different mechanism. It is unclear whether gains come from the 'thought communication' paradigm or from the specific training procedure. No matched-compute or training-signal ablation is provided."
     },
     {
       "flag": "Overclaiming from narrow benchmarks",
       "detail": "The paper claims to enable 'communication beyond language' and a paradigm for 'superhuman intelligence,' but experiments are limited to math problem-solving (MATH, GSM8K). These are well-structured tasks with clear answers — the claims far outrun the evidence."
     },
     {
       "flag": "No significance testing",
       "detail": "Claims of 'outperforming' baselines are made across many comparisons without statistical tests. Some differences are within overlapping standard deviation ranges (e.g., LLaMA-3-8B GSM8K: 68.40 ± 2.08 vs 69.20 ± 2.06)."
     },
     {
       "flag": "Standard deviations may not represent multi-run variance",
       "detail": "The ± values in Table 1 appear to be standard errors computed from the binomial distribution (binary correct/incorrect on 500 samples) rather than variance across multiple experimental runs. The paper does not clarify this, making it unclear whether the results are reproducible across runs."
     }
   ],
   "cited_papers": [
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba", "Joshua B Tenenbaum", "Igor Mordatch"],
       "year": 2023,
       "relevance": "Foundational multi-agent debate paper that THOUGHTCOMM builds upon and compares against."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Empirical analysis of multi-agent failure modes including vague message specification and inter-agent misalignment."
     },
     {
       "title": "Multiagent finetuning: Self improvement with diverse reasoning chains",
       "authors": ["Vighnesh Subramaniam", "Yilun Du", "Joshua B Tenenbaum", "Antonio Torralba", "Shuang Li", "Igor Mordatch"],
       "year": 2025,
       "arxiv_id": "2501.05707",
       "relevance": "Primary baseline — current state-of-the-art in multi-agent collaboration through finetuning."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversation",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2023,
       "arxiv_id": "2308.08155",
       "relevance": "Major multi-agent LLM framework relevant to agentic AI survey scope."
     },
     {
       "title": "MetaGPT: Meta programming for multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Xiawu Zheng", "Jonathan Chen"],
       "year": 2023,
       "arxiv_id": "2308.00352",
       "relevance": "Multi-agent collaboration framework with role-based agent design."
     },
     {
       "title": "CAMEL: Communicative agents for mind exploration of large language model society",
       "authors": ["Guohao Li", "Hasan Hammoud", "Hani Itani", "Dmitrii Khizbullin", "Bernard Ghanem"],
       "year": 2023,
       "relevance": "Multi-agent communication framework exploring role-playing agent collaboration."
     },
     {
       "title": "Large language model based multi-agents: A survey of progress and challenges",
       "authors": ["Taicheng Guo", "Xiuying Chen", "Yaqi Wang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of LLM multi-agent systems covering communication, coordination, and challenges."
     },
     {
       "title": "DSPy: Compiling declarative language model calls into self-improving pipelines",
       "authors": ["Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari"],
       "year": 2023,
       "arxiv_id": "2310.03714",
       "relevance": "LLM pipeline framework relevant to multi-agent workflow design."
     },
     {
       "title": "Encouraging divergent thinking in large language models through multi-agent debate",
       "authors": ["Tian Liang", "Zhiwei He", "Wenxiang Jiao"],
       "year": 2023,
       "arxiv_id": "2305.19118",
       "relevance": "Multi-agent debate approach for improving LLM reasoning through diverse exchanges."
     },
     {
       "title": "The llama 3 herd of models",
       "authors": ["Aaron Grattafiori", "Abhimanyu Dubey"],
       "year": 2024,
       "arxiv_id": "2407.21783",
       "relevance": "Major open-source LLM family used as base model in experiments."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning-focused LLM used as base model in experiments."
     },
     {
       "title": "Collab: Controlled decoding using mixture of agents for LLM alignment",
       "authors": ["Souradip Chakraborty", "Sujay Bhatt"],
       "year": 2025,
       "arxiv_id": "2503.21720",
       "relevance": "Token-level multi-agent collaboration approach, alternative to debate-style communication."
     }
   ]
 }