ai-research-survey

scan.json (27651B)

---

 {
   "paper": {
     "title": "Agentic Memory: Learning Unified Long-Term and Short-Term Memory Management for Large Language Model Agents",
     "authors": [
       "Yi Yu",
       "Liuyi Yao",
       "Yuexiang Xie",
       "Qingquan Tan",
       "Jiaqi Feng",
       "Yaliang Li",
       "Libing Wu"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.01885"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or archive URL is provided for the AgeMem implementation. The paper references AgentScope (Gao et al., 2025a) and Trinity-RFT (Pan et al., 2025a) frameworks used to build the system, but no repository for AgeMem itself is given. No promise or URL appears in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five evaluation benchmarks (ALFWorld, SciWorld, PDDL, BabyAI, HotpotQA) are publicly available datasets. The paper does not collect new proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states experiments were run on '8 NVIDIA RTX 4090 GPUs with 48GB memory each' (Appendix C.4), and mentions AgentScope and Trinity-RFT frameworks, but no requirements file, Dockerfile, or detailed dependency list with library versions is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "There are no step-by-step reproduction instructions. The appendix describes the algorithm and hyperparameters (K=8 rollouts, β=0.1, uniform reward weights), but there is no README or script-level guidance a researcher could follow to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 reports only point estimates (e.g., AgeMem achieves 41.07% on ALFWorld with Qwen2.5-7B). No confidence intervals, error bars, or standard deviations are reported for any main result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'AgeMem improves by 4.82 and 8.57 percentage points on average' over best baselines) with no statistical significance tests. No p-values, t-tests, or equivalent tests are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point improvements with baseline context throughout (e.g., 'relative gains of 49.59% and 23.52% over no-memory' in Section 4.2, absolute differences like '+13.9%' in ablations with both baseline and new values). Per the schema, '12% improvement over baseline (from 45% to 57%)' provides enough context for YES — the paper does this consistently."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not justify dataset split sizes used for evaluation. For HotpotQA, ~90k training questions are mentioned but the test set size used for evaluation is not explicitly stated, and no power analysis or justification for evaluation set sizes is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 2 and other tables are single-point estimates. No standard deviations, variance across seeds, or multiple-run spread measures are reported anywhere in the paper."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four baselines: LangMem, A-Mem, Mem0, and Mem0g, plus an AgeMem-noRL ablation. A no-memory baseline is also included. Baselines use their official open-source implementations (links provided in Appendix C.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2025 (LangMem, A-Mem arXiv:2502.12110, Mem0 arXiv:2504.19413), representing current state-of-the-art memory management systems for LLM agents."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 and Figure 9 present systematic ablations isolating: (1) LTM tools alone (+LT), (2) LTM tools with RL (+LT/RL), and (3) full system with STM and RL (+LT/ST/RL). A reward function ablation (All-Returns vs Answer-Only) is also provided in Tables 4 and 5."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: Success Rate (SR) for ALFWorld/SciWorld/BabyAI, Progress Rate (PR) for PDDL, LLM-as-a-Judge (J) for HotpotQA, Memory Quality (MQ) for stored memory, and token count for context efficiency."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not relevant to this benchmark-evaluation paper. All metrics are automated (task completion metrics from environments, LLM-as-a-Judge for open-ended QA)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training is done only on HotpotQA training set; evaluation is on the respective test/validation splits of each benchmark. The paper states 'fine-tuned with RL only on the HotpotQA training set and then evaluated directly on all datasets' (Section 4.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across all five benchmarks (ALFWorld, SciWorld, PDDL, BabyAI, HotpotQA) for both model backbones. Ablation figures also show per-dataset results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case studies in Appendix B compare the trained agent (successes) against a baseline that fails, but these are synthetic illustrative examples. No systematic error analysis of where AgeMem fails or analysis of failure modes on the benchmarks is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation shows that adding LTM alone to Qwen3-4B on SciWorld actually hurts performance (-4.4%, Figure 9b), and PDDL remains low even with full AgeMem. These are reported honestly without being hidden."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AgeMem 'consistently outperforms strong memory-augmented baselines across multiple LLM backbones, achieving improved task performance, higher-quality long-term memory, and more efficient context usage.' Table 2 (task performance), Figure 2 (memory quality), and Figure 3 (token counts) support these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study isolates individual components (LTM, STM, RL) through controlled single-variable manipulation, supporting causal claims like 'RL training contributes 8.53 percentage points improvement over AgeMem-noRL' (Section 4.2). The design adequately supports these component-level causal inferences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on two specific open-source model backbones (Qwen2.5-7B and Qwen3-4B) on five benchmarks, but the conclusion states results 'highlight the importance of unified, agent-centric memory policies' for 'LLM agents' generally. The title and claims do not adequately bound generalizability to the tested models and task types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions the 'fixed set of memory management tools' and limited benchmark coverage but does not discuss alternative explanations for the observed improvements—e.g., whether gains could be attributed to additional RL training compute rather than the unified memory architecture, or whether the benchmark selection favors the approach."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Qwen2.5-7B-Instruct' and 'Qwen3-4B-Instruct' as base models, and 'Qwen-Max' as the LLM evaluator. However, per the schema, marketing names without a snapshot date or API version do NOT count. No specific model checkpoint dates, snapshot identifiers, or API versions are provided for any of these models. 'Qwen-Max' is especially vague — it is a marketing name with no version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt used in AgeMem is provided verbatim in Appendix A.1, including the tool-calling format and response structure. The summarization system prompt is also fully provided, as are the LLM-as-a-Judge and Memory Quality evaluation prompt templates in Appendix C.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: K=8 rollouts, KL coefficient β=0.1, uniform reward weights (1/3 each), FILTER threshold θ=0.6, max context length 8,192 tokens, max response length 2,048 tokens. Penalty coefficients (Prounds=-1, Poverflow=-0.5) are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: the three-stage trajectory structure, tool interface (6 tools with exact signatures in Figures 6 and 7), the step-wise GRPO mechanism, and the rollout algorithms (Algorithms 1-5 in Appendix A.3) are all fully specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes how HotpotQA supporting facts are used as Stage 1 contextual information and mentions a DISTRACTORGEN function for generating distractors, but does not detail the distractor generation process. The exact evaluation subsets (sizes, selection criteria) for each benchmark are not documented. The data pipeline from raw benchmarks to actual training/evaluation instances is underspecified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present after the conclusion, discussing the fixed tool set and limited benchmark coverage as areas for future extension."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is generic: it notes that 'broader coverage of tasks and environments may further strengthen the empirical understanding' but does not identify specific threats to validity for the current results (e.g., benchmark contamination risks, LLM judge reliability, reward hacking possibilities)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. For example, it does not note that results are limited to two specific Qwen model families, that the RL training was done only on HotpotQA, or that the 5.1% token reduction may not be practically significant."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The underlying evaluation logs, model outputs, and raw trajectory data are not released. Only the aggregated performance numbers in tables and figures are available. Public benchmarks (HotpotQA, ALFWorld, etc.) are accessible but the model's specific outputs cannot be verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The experimental datasets are all established public benchmarks with well-documented collection procedures from their original papers. The paper references original dataset sources and describes their structure (Appendix C.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper evaluates on standard benchmarks and has no human participants, so recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the three-stage trajectory structure is described algorithmically, the DISTRACTORGEN function for generating distractors is mentioned but not fully specified. The exact evaluation subsets and how benchmark data is converted into the three-stage format are not fully documented. The data pipeline has unexplained steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section or funding disclosure in the paper. The affiliation includes Alibaba Group, but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Yi Yu, Liuyi Yao, Yuexiang Xie, Yaliang Li are from Alibaba Group; others are from Wuhan University. The paper evaluates a system built using Alibaba's AgentScope framework."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Several authors are from Alibaba Group, and the system is built using Alibaba's AgentScope framework and evaluated using Qwen models (also from Alibaba). The organization developing the evaluated technology is directly represented among the authors, creating a non-independent relationship."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement anywhere in the paper. The Alibaba affiliation is listed but no explicit declaration of financial interests or potential conflicts is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-7B-Instruct and Qwen3-4B-Instruct as base models but does not state their training data cutoff dates. This is relevant because HotpotQA (2018) and BabyAI (2018) are long-established benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether benchmark test examples (from HotpotQA, ALFWorld, etc.) appeared in Qwen model training data. Given that these are established public benchmarks predating Qwen model training, contamination risk exists but is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (2018), ALFWorld (2020), SciWorld (2022), and BabyAI (2018) all predate the Qwen models' training cutoff. The paper does not discuss whether these benchmarks were present in the training data, which could inflate base model performance."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study; it is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; ethics approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses token efficiency (average tokens per episode in Figure 3, ~2,117-2,310 tokens) but does not report API costs, wall-clock inference time per example, or cost per episode. The use of Qwen-Max as LLM evaluator implies API costs that are not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '8 NVIDIA RTX 4090 GPUs with 48GB memory each' (Appendix C.4), specifying hardware. However, no training wall-clock time, total GPU-hours, or total compute budget is stated. Knowing the GPU model without knowing how long training took does not constitute a stated compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "AgeMem achieves the highest average performance on both Qwen2.5-7B-Instruct (41.96%) and Qwen3-4B-Instruct (54.31%), outperforming all baselines across five datasets with relative gains of 49.59% and 23.52% over no-memory baselines.",
       "evidence": "Table 2 in Section 4.2 shows per-dataset and average performance numbers for both model backbones across all methods. AgeMem outperforms all four baselines in average score on both models.",
       "supported": "moderate"
     },
     {
       "claim": "RL training contributes 8.53 and 8.72 percentage point improvements over AgeMem-noRL on Qwen2.5-7B-Instruct and Qwen3-4B-Instruct respectively.",
       "evidence": "Derived from Table 2: AgeMem (41.96%) vs AgeMem-noRL (33.43%) for Qwen2.5-7B, and AgeMem (54.31%) vs AgeMem-noRL (45.59%) for Qwen3-4B.",
       "supported": "moderate"
     },
     {
       "claim": "AgeMem achieves higher memory quality (MQ) than all baselines: 0.533 on Qwen2.5-7B and 0.605 on Qwen3-4B.",
       "evidence": "Figure 2 shows MQ scores for all methods. However, the MQ metric is measured using Qwen-Max as LLM judge, which introduces potential bias since the system was trained using the same model family.",
       "supported": "moderate"
     },
     {
       "claim": "AgeMem reduces prompt token usage compared to RAG-based variants by 3.1% (Qwen2.5) and 5.1% (Qwen3-4B) on HotpotQA.",
       "evidence": "Figure 3 shows average token counts. The absolute reduction is small (2,117 vs 2,186 for Qwen2.5-7B), and statistical significance is not tested.",
       "supported": "weak"
     },
     {
       "claim": "The multi-component reward function (All-Returns) leads to faster convergence and higher final performance than task-only reward (Answer-Only).",
       "evidence": "Figures 5 and 10 show convergence curves; Tables 4 and 5 show that All-Returns achieves higher J scores and much better MQ scores on both model backbones.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "AgeMem proposes a unified framework for jointly managing long-term and short-term memory in LLM agents via six tool-based operations (ADD, UPDATE, DELETE, RETRIEVE, SUMMARY, FILTER) trained with a three-stage progressive reinforcement learning strategy and a step-wise GRPO algorithm. Evaluated on five long-horizon benchmarks (ALFWorld, SciWorld, PDDL, BabyAI, HotpotQA) with two Qwen model backbones, AgeMem outperforms four contemporary memory management baselines (LangMem, A-Mem, Mem0, Mem0g) by 4.82-8.57 percentage points on average. Ablation studies confirm that RL training and the combination of LTM and STM tools each contribute meaningfully to performance. Token efficiency improvements from learned STM management over RAG are modest (3-5% reduction in average prompt tokens).",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results in Table 2 are single point estimates with no standard deviations, confidence intervals, or error bars reported. Given that RL training involves stochastic rollouts (K=8 per task), variance across training seeds or evaluation runs could be substantial, and a 4-5 percentage point difference over baselines may not be statistically significant."
     },
     {
       "flag": "Alibaba-affiliated authors evaluating Alibaba tools",
       "detail": "Multiple authors are from Alibaba Group, the system is built on Alibaba's AgentScope framework, evaluated with Qwen models (Alibaba), and the LLM judge is Qwen-Max (also Alibaba). No competing interests or independence statement is provided. This creates a potential for confirmation bias in experimental design and metric selection."
     },
     {
       "flag": "LLM-as-judge circularity",
       "detail": "Memory Quality (MQ) is measured using Qwen-Max as the evaluator. The base models (Qwen2.5-7B, Qwen3-4B) are from the same family, and the training uses RL rewards partially computed by LLM judges. Using the same model family for both training rewards and evaluation metrics introduces circularity that could inflate AgeMem's apparent MQ advantage."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "HotpotQA (2018), ALFWorld (2020), BabyAI (2018), and SciWorld (2022) are all long-established public benchmarks. The Qwen models' training data cutoffs are not stated, so it is possible the base models have seen these benchmarks, inflating the no-memory baseline and confounding the comparison."
     },
     {
       "flag": "No code release",
       "detail": "The AgeMem implementation is not released. Only the third-party frameworks (AgentScope, Trinity-RFT) are available, making reproduction of the specific AgeMem training procedure very difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "A-Mem: Agentic Memory for LLM Agents",
       "authors": [
         "Wujiang Xu",
         "Zujie Liang",
         "Kai Mei",
         "Hang Gao",
         "Juntao Tan",
         "Yongfeng Zhang"
       ],
       "year": 2025,
       "arxiv_id": "2502.12110",
       "relevance": "Directly compared baseline for LLM agent long-term memory management using Zettelkasten-inspired design."
     },
     {
       "title": "Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory",
       "authors": [
         "Prateek Chhikara",
         "Dev Khant",
         "Saket Aryan",
         "Taranjeet Singh",
         "Deshraj Yadav"
       ],
       "year": 2025,
       "arxiv_id": "2504.19413",
       "relevance": "Directly compared baseline for scalable LTM management in AI agents."
     },
     {
       "title": "ReSum: Unlocking Long-Horizon Search Intelligence via Context Summarization",
       "authors": [
         "Xixi Wu",
         "Kuan Li",
         "Yida Zhao"
       ],
       "year": 2025,
       "arxiv_id": "2509.13313",
       "relevance": "Prior work on short-term memory management through context summarization, used as a comparison point."
     },
     {
       "title": "AgentBoard: An Analytical Evaluation Board of Multi-Turn LLM Agents",
       "authors": [
         "Ma Chang",
         "Junlei Zhang",
         "Zhihao Zhu",
         "Cheng Yang",
         "Yujiu Yang"
       ],
       "year": 2024,
       "relevance": "Benchmark used (PDDL) and citation for multi-turn LLM agent evaluation methodology."
     },
     {
       "title": "Memory-R1: Enhancing Large Language Model Agents to Manage and Utilize Memories via Reinforcement Learning",
       "authors": [
         "Sikuan Yan",
         "Xiufeng Yang",
         "Zuchao Huang"
       ],
       "year": 2025,
       "arxiv_id": "2508.19828",
       "relevance": "Related concurrent work on using RL for LLM agent memory management."
     },
     {
       "title": "Trinity-RFT: A General-Purpose and Unified Framework for Reinforcement Fine-Tuning of Large Language Models",
       "authors": [
         "Xuchen Pan",
         "Yanxi Chen",
         "Yushuo Chen"
       ],
       "year": 2025,
       "arxiv_id": "2505.17826",
       "relevance": "The RL training framework used in AgeMem experiments."
     },
     {
       "title": "DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models",
       "authors": [
         "Zhihong Shao",
         "Peiyi Wang",
         "Qihao Zhu"
       ],
       "year": 2024,
       "arxiv_id": "2402.03300",
       "relevance": "Source of the GRPO algorithm used for RL training in AgeMem."
     },
     {
       "title": "HotpotQA: A Dataset for Diverse, Explainable Multi-Hop Question Answering",
       "authors": [
         "Zhilin Yang",
         "Peng Qi",
         "Saizheng Zhang",
         "Yoshua Bengio",
         "William Cohen"
       ],
       "year": 2018,
       "relevance": "Primary training and evaluation benchmark used in AgeMem experiments."
     },
     {
       "title": "ALFWorld: Aligning Text and Embodied Environments for Interactive Learning",
       "authors": [
         "Mohit Shridhar",
         "Xingdi Yuan",
         "Marc-Alexandre Cote"
       ],
       "year": 2020,
       "arxiv_id": "2010.03768",
       "relevance": "Key evaluation benchmark for embodied long-horizon agent tasks."
     },
     {
       "title": "How Memory Management Impacts LLM Agents: An Empirical Study of Experience-Following Behavior",
       "authors": [
         "Zidi Xiong",
         "Yuping Lin",
         "Wenya Xie"
       ],
       "year": 2025,
       "arxiv_id": "2505.16067",
       "relevance": "Empirical study on memory management impacts for LLM agents, directly relevant to survey scope."
     },
     {
       "title": "A Survey on the Memory Mechanism of Large Language Model-based Agents",
       "authors": [
         "Zeyu Zhang",
         "Quanyu Dai",
         "Xiaohe Bo"
       ],
       "year": 2025,
       "relevance": "Survey of memory mechanisms in LLM agents, highly relevant as a taxonomy reference."
     },
     {
       "title": "Zep: A Temporal Knowledge Graph Architecture for Agent Memory",
       "authors": [
         "Preston Rasmussen",
         "Pavlo Paliychuk",
         "Travis Beauvais"
       ],
       "year": 2025,
       "arxiv_id": "2501.13956",
       "relevance": "Competing approach to LLM agent memory using temporal knowledge graphs."
     },
     {
       "title": "AgentScope 1.0: A Developer-Centric Framework for Building Agentic Applications",
       "authors": [
         "Dawei Gao",
         "Zitao Li",
         "Yuexiang Xie"
       ],
       "year": 2025,
       "arxiv_id": "2508.16279",
       "relevance": "The agent scaffolding framework used to build AgeMem, relevant to agentic system infrastructure research."
     }
   ]
 }