ai-research-survey

scan.json (24503B)

---

 {
   "paper": {
     "title": "LatentMem: Customizing Latent Memory for Multi-Agent Systems",
     "authors": ["Muxin Fu", "Guibin Zhang", "Xiangyuan Xue", "Yafu Li", "Zefeng He", "Siyuan Huang", "Xiaoye Qu", "Yu Cheng", "Yang Yang"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.03036"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided in the abstract header: https://github.com/KANABOON1/LatentMem"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six benchmarks used (TriviaQA, PopQA, KodCode, BigCodeBench, StrategyQA, PDDL) are publicly available datasets. Descriptions in Appendix B.1."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 lists training hyperparameters and mentions Flash Attention, vLLM, DeepSpeed, but no requirements.txt, Dockerfile, or specific library versions are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README commands, or scripts for replicating experiments are described in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 4 report only point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LatentMem outperforms baselines based solely on comparing numbers. No statistical significance tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 4 report percentage point improvements over baselines with both absolute values and deltas (e.g., '76.51↑16.20'), providing sufficient context for effect size."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of evaluation examples used from each benchmark, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures reported across runs. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven memory baselines compared: Voyager, Generative, JoyAgent, MetaGPT, ChatDev, OAgents, G-Memory, plus no-memory and MARTI fine-tuning baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent works: G-Memory (2025), JoyAgent (2025), OAgents (2025), MARTI (2025). All are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 and Figure 6 (Right) present ablation removing role profiles ('without role') and experience bank updates ('without experience'), showing contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Evaluates accuracy across 6 benchmarks in different domains, plus time cost and token cost analysis in Section 5.3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs. All evaluation is automated (accuracy, pass rates, reward scores)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Explicit held-in/held-out split: TriviaQA, KodCode, StrategyQA, PopQA are in-domain; BigCodeBench and PDDL are out-of-domain. CAMEL and DyLAN are unseen MAS frameworks (Section 5.1, Appendix B.4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-benchmark, per-MAS-framework results across all 6 datasets and 4 frameworks."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.7 and Figure 7 present a case study analyzing common error patterns (step repetition, disobey task specification, reasoning-action mismatch) and how LatentMem addresses them."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study shows performance drops when components are removed (e.g., 6.45% drop on MacNet KodCode without role profiles). Some baselines hurt performance vs no-memory (e.g., ChatDev on AutoGen TriviaQA: -2.97%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'up to 19.36%' improvement — Table 1 DyLAN PopQA shows exactly 19.36%. Claims of 50% fewer tokens and ~2/3 inference time are supported by Figure 3. All abstract claims are backed by results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by ablation studies (Section 5.6) with controlled single-variable manipulation (removing role profiles, disabling experience bank updates)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly separates held-in vs held-out benchmarks and seen vs unseen MAS frameworks, and reports results separately for each category. Claims are framed within the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the performance gains. For example, no consideration of whether improvements could be due to increased compute, data augmentation effects, or other confounds beyond the memory mechanism."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy/pass rates on specific benchmarks and frames claims at that level ('performance gain of up to 19.36% over vanilla settings'). No broader framing beyond what was measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model identifiers provided: 'Qwen/Qwen3-4B-Instruct-2507' and 'meta-llama/Llama-3.1-8B-Instruct' (Section 5.1). Embedding model: 'all-MiniLM-L6-v2'."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D provides full prompt templates for all agent roles across CAMEL and AutoGen frameworks, with actual prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 provides comprehensive hyperparameter settings including learning rate (1e-5), temperature (1.0 train, 0.0 eval), clipping epsilon (0.2), LoRA settings (r=16, alpha=32), batch sizes, and more."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.3 describes each MAS framework setup in detail: AutoGen (B.3.1), MacNet (B.3.2), CAMEL (B.3.3), DyLAN (B.3.4), including agent counts, topologies, and coordination mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4.2 describes training trajectory collection: which MAS frameworks and datasets were used, total trajectory count (40,580), and how trajectories were processed for parametric vs non-parametric baselines."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No limitations section. The paper has an 'Impact Statement' with ethical considerations and societal implications, but no discussion of methodological limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity discussed. The Impact Statement addresses only ethical and societal concerns, not study-specific methodological threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries. The paper does not state what the results do NOT show, what settings are excluded, or what claims the authors are NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (trajectories, per-example results) is made available for independent verification. Only aggregated results are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4.2 describes training trajectory collection: 'we collect data on all in-domain datasets using the training splits and in-distribution MAS,' yielding 40,580 trajectories."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.4.2 documents the pipeline: collect trajectories from in-domain datasets using training splits → store in experience bank → train memory composer via LMPO on same training data."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section with grants or sponsors is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Tongji University, Shanghai AI Laboratory, National University of Singapore, Chinese University of Hong Kong, Nanjing University, Shanghai Jiao Tong University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosed, so independence cannot be assessed. Authors include Shanghai AI Laboratory affiliates, but no explicit conflict statement."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates stated for Qwen3-4B-Instruct-2507 or Llama-3.1-8B-Instruct, despite evaluating these models on public benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (TriviaQA, StrategyQA, PopQA, etc.) appeared in the LLM training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TriviaQA (2017), StrategyQA (2021), PopQA (2023) all predate the models' training. No contamination risk discussion despite high likelihood of exposure."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 and Figures 3 and 8 report both time costs (in seconds) and token costs for all methods across multiple benchmarks and frameworks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total training compute budget (GPU hours, hardware used, training time) is reported despite training the memory composer with LMPO."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds. All tables report single point estimates without seed variation analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing reported results is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Sensitivity analysis in Section 5.6 varies L' and K, but no overall hyperparameter search budget is reported. The total configurations tried to arrive at the final settings are not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 and Figure 6 show sensitivity analysis for L' with performance curves, justifying the choice of L'=8 as 'balancing accuracy and computational cost.'"
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors implement and evaluate their own system against their implementations of baselines without discussing this potential bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 8 explicitly plot performance vs time cost and performance vs token cost, showing LatentMem's efficiency-performance trade-off relative to baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure what the paper claims. For example, no analysis of whether TriviaQA accuracy truly reflects 'knowledge QA capability' in multi-agent settings."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates LatentMem across 4 different MAS frameworks (AutoGen, MacNet, CAMEL, DyLAN), showing results hold across different scaffolding architectures. Table 1 systematically compares across frameworks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. TriviaQA (2017), StrategyQA (2021), and PopQA (2023) all predate the models used, creating significant contamination risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the retrieved experience trajectories or evaluation setup leak answer information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training trajectories and test examples. Training and test data come from splits of the same benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "LatentMem improves state-of-the-art MAS by up to 16.20% on knowledge QA (TriviaQA) and 18.45% on code generation (KodCode) tasks.",
       "evidence": "Table 1 shows AutoGen+TriviaQA: 76.51 vs 60.31 no-memory (+16.20%). Table 4 shows AutoGen+KodCode with Llama-3.1-8B: 65.90 vs 47.45 (+18.45%).",
       "supported": "strong"
     },
     {
       "claim": "LatentMem uses 50% fewer tokens and reduces inference time to ~2/3 compared to mainstream memory designs.",
       "evidence": "Figure 3 shows token and time costs. AutoGen+KodCode: LatentMem uses 1.76M tokens vs 3.04-3.64M for baselines. Time costs similarly reduced.",
       "supported": "moderate"
     },
     {
       "claim": "LatentMem generalizes to out-of-domain datasets (PDDL: +7.10%) and unseen MAS frameworks (CAMEL: +7.90% average over vanilla).",
       "evidence": "Table 1 AutoGen+PDDL: 23.49 vs 16.39 (+7.10%). CAMEL average held-in: 61.12 vs 53.22 (+7.90%).",
       "supported": "strong"
     },
     {
       "claim": "LatentMem consistently outperforms direct agent backbone fine-tuning (MARTI) across all settings.",
       "evidence": "Table 2 shows LatentMem vs MARTI on KodCode and TriviaQA across AutoGen and MacNet, with LatentMem winning all four comparisons (up to +11.73% on AutoGen TriviaQA).",
       "supported": "moderate"
     },
     {
       "claim": "LatentMem generates role-specific latent memories that form well-separated clusters even on unseen frameworks and out-of-domain data.",
       "evidence": "Figures 4 and 9 show t-SNE visualizations with clear role-based clustering on both in-domain (AutoGen+KodCode) and out-of-domain (CAMEL+BigCodeBench) settings.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LatentMem proposes a learnable latent memory framework for multi-agent systems that addresses memory homogenization and information overload. It achieves performance gains of up to 19.36% over vanilla MAS settings across 6 benchmarks and 4 frameworks, while using approximately 50% fewer tokens than competing memory architectures. The framework generalizes to unseen MAS frameworks and out-of-domain benchmarks, and outperforms direct backbone fine-tuning (MARTI) under matched compute budgets.",
   "red_flags": [
     {
       "flag": "No variance or multiple-run reporting",
       "detail": "All results in Tables 1 and 4 are single point estimates with no standard deviations, error bars, or indication of how many runs produced them. Given LLM output stochasticity and MAS interaction variability, single-run results may not be stable."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Performance improvements are claimed based on raw number comparisons without any statistical testing. Some improvements (e.g., 0.04% for MetaGPT on AutoGen TriviaQA) could easily be within noise."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity discussion. The Impact Statement covers only ethical/societal considerations, not methodological weaknesses."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "TriviaQA (2017), StrategyQA (2021), and PopQA (2023) all predate the training of both Qwen3 and Llama-3.1. The paper does not discuss whether the backbone models have seen these benchmarks, which would inflate all results including baselines."
     },
     {
       "flag": "Training compute budget not reported",
       "detail": "The memory composer is trained with LMPO on 40,580 trajectories but no GPU hours, hardware specs, or training wall-clock time are reported, making cost-effectiveness claims incomplete."
     }
   ],
   "cited_papers": [
     {
       "title": "Agentverse: Facilitating multi-agent collaboration and exploring emergent behaviors",
       "authors": ["Weize Chen", "Yusheng Su", "Jingwei Zuo"],
       "year": 2023,
       "relevance": "Multi-agent collaboration framework relevant to MAS evaluation and design."
     },
     {
       "title": "MetaGPT: Meta programming for a multi-agent collaborative framework",
       "authors": ["Sirui Hong", "Mingchen Zhuge", "Jonathan Chen"],
       "year": 2023,
       "relevance": "Major MAS framework used as both baseline and evaluation setting."
     },
     {
       "title": "ChatDev: Communicative agents for software development",
       "authors": ["Chen Qian", "Wei Liu", "Hongzhang Liu"],
       "year": 2024,
       "relevance": "MAS framework for code generation, used as baseline memory architecture."
     },
     {
       "title": "AutoGen: Enabling next-gen LLM applications via multi-agent conversations",
       "authors": ["Qingyun Wu", "Gagan Bansal", "Jieyu Zhang"],
       "year": 2024,
       "relevance": "Core MAS framework used for evaluation, one of four tested frameworks."
     },
     {
       "title": "CAMEL: Communicative agents for 'mind' exploration of large language model society",
       "authors": ["Guohao Li", "Hasan Hammoud", "Hani Itani"],
       "year": 2023,
       "relevance": "Role-playing MAS framework used as unseen evaluation framework."
     },
     {
       "title": "Voyager: An open-ended embodied agent with large language models",
       "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang"],
       "year": 2023,
       "relevance": "Single-agent memory baseline adapted for multi-agent comparison."
     },
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph O'Brien", "Carrie Jun Cai"],
       "year": 2023,
       "relevance": "Foundational generative agent memory architecture used as baseline."
     },
     {
       "title": "MARTI: A framework for multi-agent LLM systems reinforced training and inference",
       "authors": ["Kaiyan Zhang", "Runze Liu", "Xuekai Zhu"],
       "year": 2025,
       "relevance": "Multi-agent fine-tuning baseline compared under matched compute budgets."
     },
     {
       "title": "BigCodeBench",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "relevance": "Code generation benchmark used as out-of-domain evaluation dataset."
     },
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Mert Cemri", "Melissa Z Pan", "Shuyi Yang"],
       "year": 2025,
       "relevance": "Analysis of MAS failure modes, motivates the memory homogenization problem addressed by LatentMem."
     },
     {
       "title": "G-Memory: Tracing hierarchical memory for multi-agent systems",
       "authors": ["Guibin Zhang", "Muxin Fu", "Guancheng Wan"],
       "year": 2025,
       "relevance": "Multi-agent hierarchical memory baseline; shared authorship with LatentMem."
     },
     {
       "title": "KodCode: A diverse, challenging, and verifiable synthetic dataset for coding",
       "authors": ["Zhangchen Xu", "Yang Liu", "Yueqin Yin"],
       "year": 2025,
       "relevance": "Synthetic coding benchmark used for in-domain evaluation."
     }
   ]
 }