ai-research-survey

scan.json (24759B)

---

 {
   "paper": {
     "title": "Dual Latent Memory for Visual Multi-agent System",
     "authors": ["Xinlei Yu", "Chengming Xu", "Zhangquan Chen", "Bo Yin", "Cheng Yang", "Yongbo He", "Yihao Hu", "Jiangning Zhang", "Cheng Tan", "Xiaobin Hu", "Shuicheng Yan"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.00471"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper identifies a 'scaling wall' in visual multi-agent systems where increasing agent turns degrades performance while exponentially inflating token costs. The proposed L2-VMAS framework with dual latent memories (perception and thinking) improves average accuracy by 2.7-5.4% while reducing token usage by 21.3-44.8% across five VLM backbones. The method demonstrates consistent gains across four model sizes (2B-32B) and six multi-agent topologies, and shows strong scalability up to 10 agent turns where baselines degrade below single-agent performance.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Codes: https://github.com/YU-deep/L2-VMAS' providing a GitHub repository URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (MMBench, MMStar, RealWorldQA, SimpleVQA, MuirBench, BLINK, MVBench, LVBench) and trains on the public GQA dataset. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA H200 141G GPUs' but does not provide requirements.txt, Dockerfile, or detailed library versions for environment recreation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but the paper itself contains no README-level reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results tables (Tables 1-3, 5-10) report only point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims L2-VMAS outperforms baselines across multiple settings but provides no statistical significance tests — comparisons are based solely on point estimate differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '2.7-5.4% accuracy improvement' and '21.3-44.8% token reduction', with absolute values in tables showing both baseline and proposed method performance."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of benchmarks or their sizes. No power analysis or discussion of whether the benchmark sizes are sufficient to detect the claimed improvements."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against both single-agent and multi-agent (VMAS with text-based communication) baselines across all experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines use contemporary VLMs including Qwen3-VL (2025), GLM-4.1V-Thinking (2025), InternVL-3.5-8B (2025), and LLaVA-OV-1.5-8B (2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides ablation removing individual components: triggering, attribution, perception memory, and thinking memory, showing each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy and total token usage across all experiments, evaluating effectiveness and efficiency jointly."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here — the paper evaluates on established automated visual QA benchmarks with ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses the GQA dataset with 'no exposure to the test benchmarks' (Section 4.1). Table 10 also shows generalization to unseen benchmarks."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (Tables 1-3), per model size (Table 2), per multi-agent structure (Table 3), and per task type (perception/thinking/mixed in Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 discusses three cells where L2-VMAS slightly underperforms VMAS (LLaVA-OV on MMBench, Qwen3-VL-2B on MMBench and RealWorldQA), attributing this to performance ceiling saturation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cases where L2-VMAS slightly degrades performance (Table 1: LLaVA-OV-1.5-8B on MMBench drops 0.3%, Table 2: Qwen3-VL-2B drops on two benchmarks) and discusses why."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '2.7-5.4% accuracy improvement' and '21.3-44.8% token reduction' are directly supported by Table 1 results across five backbones."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'dual latent memory improves performance') are supported by controlled ablation studies (Table 6) that remove individual components and measure the impact, constituting adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims applicability to 'Visual Multi-agent System' generally, but results are limited to visual QA tasks on specific benchmarks. The paper does not test on open-ended generation, visual dialogue, or other VMAS application domains (e.g., embodied agents, robotics). The generalization claims in the title are broader than the evidence."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the improvements. For example, the gains could partly be due to the additional learnable parameters in the memory modules rather than the dual memory design per se. No threats-to-validity or alternative explanations section exists."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures accuracy on QA benchmarks and token usage, and frames claims in terms of these specific metrics rather than broader unsubstantiated constructs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: GLM-4.1V-9B-Thinking, InternVL-3.5-8B, LLaVA-OV-1.5-8B, Qwen3-VL-8B-Thinking/Instruct, and specific size variants (2B/4B/8B/32B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full prompt used for perception/thinking extraction via GPT-5.1 is provided in Appendix B.1. System prompts for the multi-agent setup are referenced in the multi-agent settings from prior work."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides comprehensive three-stage training hyperparameters including learning rates, clip range, batch sizes, PPO parameters. Section 4.1 states W=16, λ=0.5, L=8, N=50."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-agent structures (six topologies) are described in detail in Section 4.1 and Appendix D.1 with illustrations in Figure 9. The memory synthesis and orchestration pipeline is thoroughly described in Sections 3.2-3.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how visual inputs are processed through hierarchical downsampling (Equation 4-5), how thinking trajectories are chunked at entropy boundaries (Equation 6), and training data comes from GQA dataset."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section in the paper. The conclusion (Section 5) does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. It does not bound its claims to the specific QA benchmark setting."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-example predictions, logs) is made available. Only aggregated results are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Training uses the public GQA dataset. Evaluation benchmarks are all publicly available with citations provided (MMBench, MMStar, RealWorldQA, SimpleVQA, etc.)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage training pipeline is documented in detail (Table 4, Appendix C.5), with clear progression from memory synthesis to orchestration to joint training."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: NUS, FDU, THU, DeepWisdom, ZJU, HNU, Shanghai AI Lab."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed. Authors are from both academic institutions and DeepWisdom/Shanghai AI Lab (industry)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained VLMs (Qwen3-VL, GLM-4.1V, InternVL, LLaVA) without stating their training data cutoff dates."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained VLM backbones may have seen the benchmark data during their pre-training."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmarks like MMBench (2024) and others are publicly available and could be in the training data of the VLMs used. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Token usage is reported for all experiments as a primary metric, showing both absolute token counts and percentage reductions (Tables 1-3, 7-9)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states '8 NVIDIA H200 141G GPUs'. Table 4 provides training steps (100k/80k/50k) across three stages. However, total GPU hours are not explicitly stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments reported. All results appear to be from single runs without seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Sensitivity analyses (Tables 7-9) explore hyperparameter values but no search budget is reported. It is unclear how the final values (W=16, λ=0.5, L=8) were selected beyond what is shown."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Tables 7-9 show sensitivity analyses across multiple values for W, λ, and L, and the selected values correspond to the best or near-best accuracy-efficiency tradeoff shown in these tables."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 5 backbones × 4 benchmarks × multiple structures but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their L2-VMAS against their own implementation of the VMAS baseline without acknowledging potential bias in baseline implementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Token usage (a compute proxy) is reported alongside accuracy for every experiment, and the paper explicitly frames the accuracy-cost tradeoff. Tables 7-9 show how different parameter settings affect both metrics."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple benchmarks but does not discuss whether these benchmarks adequately measure the claimed 'comprehensive visual abilities' or whether improvements on QA translate to real VMAS applications."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 evaluates across six different multi-agent structures (linear, layered, centralized, random, complete, dynamic) using the same backbone, and comparisons use the same structure for VMAS vs L2-VMAS."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the VLM backbones were trained on data that could include the benchmark problems."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup provides information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training data (GQA) and test benchmarks, or between the VLM pre-training data and test sets."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "L2-VMAS improves average accuracy by 2.7-5.4% over text-based VMAS across five VLM backbones while reducing token usage by 21.3-44.8%.",
       "evidence": "Table 1 shows results across GLM-4.1V-9B-Thinking (+4.3%, -44.8%), InternVL-3.5-8B (+2.7%, -25.4%), LLaVA-OV-1.5-8B (+3.2%, -21.3%), Qwen3-VL-8B-Instruct (+3.7%, -23.8%), and Qwen3-VL-8B-Thinking (+5.4%, -43.6%).",
       "supported": "moderate"
     },
     {
       "claim": "Existing VMAS suffer from a 'scaling wall' where performance degrades after 3 agent turns and drops below single-agent baseline by turn 6.",
       "evidence": "Figure 2 and Section 2.1 show accuracy peaks at 86.6 at turn 3 then falls below single-agent baseline (84.8) by turn 6, on MMBench with Qwen3-VL-8B-Thinking.",
       "supported": "moderate"
     },
     {
       "claim": "L2-VMAS achieves consistent improvements across all six multi-agent topologies (2.7-6.3% gains).",
       "evidence": "Table 3 shows gains across linear (+2.7/3.7%), layered (+2.8/3.8%), centralized (+4.5/6.3%), random (+3.1/5.1%), complete (+3.1/5.5%), and dynamic (+3.7/5.4%) structures.",
       "supported": "moderate"
     },
     {
       "claim": "L2-VMAS scales effectively to 10 agent turns with 13.6-15.1% improvement over single-agent, while baselines degrade.",
       "evidence": "Figure 6 and Section 4.3 show continued improvement at 10 turns for L2-VMAS while VMAS declines below single-agent level on RealWorldQA.",
       "supported": "moderate"
     },
     {
       "claim": "Each component (triggering, attribution, perception memory, thinking memory) contributes to the overall performance.",
       "evidence": "Table 6 ablation study shows removing attribution causes the largest drop (-2.3 to -2.8%) and removing thinking memory causes -1.5 to -3.2% drops.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty reporting",
       "detail": "All experiments report single-run point estimates without standard deviations, confidence intervals, or multi-seed results. Given the stochastic nature of both RL training and VLM inference, the claimed improvements (often 2-3%) could be within noise."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Dozens of comparisons are made across backbones, model sizes, structures, and benchmarks, but no statistical tests are performed. Small differences (e.g., 0.5% in Table 1 InternVL MMBench) are presented as meaningful without any significance verification."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or broader impact section, despite being a systems paper with many design choices that could affect generalization."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Pre-trained VLMs may have seen benchmark data during pre-training. Since the approach builds on these VLMs and claims to improve their performance on specific benchmarks, contamination could differentially affect results."
     },
     {
       "flag": "Additional parameters not controlled for",
       "detail": "L2-VMAS introduces learnable compression modules, gating networks, and refinement compressors — additional parameters that could explain improvements independent of the dual memory design. The ablation removes entire memory types but does not control for parameter count."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes of multi-agent LLM systems, directly relevant to understanding agentic workflow reliability."
     },
     {
       "title": "Scaling large language model-based multi-agent collaboration",
       "authors": ["Chen Qian"],
       "year": 2025,
       "relevance": "Studies scaling properties of LLM multi-agent collaboration, relevant to understanding agent coordination challenges."
     },
     {
       "title": "Large language models miss the multi-agent mark",
       "authors": ["Emanuele La Malfa"],
       "year": 2025,
       "relevance": "Evaluates multi-agent LLM performance, relevant to understanding limitations of LLM-based multi-agent systems."
     },
     {
       "title": "Thought communication in multiagent collaboration",
       "authors": ["Yubo Zheng"],
       "year": 2025,
       "relevance": "Explores latent thought communication between agents, directly relevant to agentic collaboration mechanisms."
     },
     {
       "title": "Cache-to-cache: Direct semantic communication between large language models",
       "authors": ["Tao Fu"],
       "year": 2025,
       "arxiv_id": "2510.03215",
       "relevance": "Proposes direct latent communication between LLMs, relevant to inter-agent communication efficiency."
     },
     {
       "title": "Latent collaboration in multi-agent systems",
       "authors": ["Jiaming Zou"],
       "year": 2025,
       "arxiv_id": "2511.20639",
       "relevance": "Studies latent-space collaboration between agents, directly relevant to non-textual agent communication."
     },
     {
       "title": "Cowpox: Towards the immunity of VLM-based multi-agent systems",
       "authors": ["Yixuan WU"],
       "year": 2025,
       "relevance": "Addresses security and robustness of VLM-based multi-agent systems, relevant to AI safety in agentic workflows."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian"],
       "year": 2023,
       "relevance": "Foundational work on multi-agent LLM collaboration for software engineering tasks."
     },
     {
       "title": "Improving factuality and reasoning in language models through multiagent debate",
       "authors": ["Yilun Du"],
       "year": 2023,
       "relevance": "Proposes multi-agent debate for improving LLM reasoning, relevant to multi-agent collaboration paradigms."
     },
     {
       "title": "Towards a science of scaling agent systems",
       "authors": ["Yoonsang Kim"],
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Investigates scaling laws for agent systems, relevant to understanding how multi-agent systems scale."
     }
   ]
 }