ai-research-survey

scan.json (21532B)

---

 {
   "paper": {
     "title": "Cost and accuracy of long-term memory in Distributed Multi-Agent Systems based on Large Language Models",
     "authors": ["Benedict Wolff", "Jacopo Bennati"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.07978"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/wolffbe/dmas-memory. The abstract states 'the source code of the testbed, experimental results as well as notebooks for analyzing and evaluating the results are available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states experimental results are available at the GitHub repository. The benchmark used (LoCoMo) is publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.4 (Table 7) lists detailed component versions including Docker 4.48.0, Python 3.11-slim, Neo4j 5.26, Ubuntu 22.04.3 LTS, and hardware specs (12th-gen Intel Core i5-12450H). Docker-based setup provides reproducible environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes a Makefile that 'automates the testbed setup, and executes the analysis Jupyter notebook.' The repository contains the testbed code, Makefile, and analysis notebooks."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 reports 95% Wilson confidence intervals for accuracy rates (e.g., Graphiti unconstrained: [0.074, 0.162])."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports two-proportion z-tests with z-statistics and p-values for all pairwise comparisons. The paper correctly concludes no significant differences at α = 0.05."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage differences with baseline context throughout (e.g., 'Graphiti was 3.6% more accurate than mem0', 'mem0 was 86.5% faster during the loading phase', cost differentials of 40.2%). Raw numbers and percentages are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "N=199 questions from one conversation of the LoCoMo benchmark. The paper states 'a robust sample size with statistical relevance was ensured' but provides no power analysis or formal justification for why 199 questions from a single conversation is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across runs. The experiments appear to be single-run only — no mention of multiple experimental runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two memory frameworks (mem0 and Graphiti) are compared against each other across two network conditions, providing a comparative evaluation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Both mem0 and Graphiti are contemporary frameworks. The paper notes mem0 'recently secured $24 million USD in funding' and Neo4j (Graphiti's backend) raised $325M in Series F, both in 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The system has multiple components (coordinator, memory agent, responder, SLM for tool-calling, LLM for answering) but none are ablated to understand individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics reported: accuracy, IDK rate, CPU usage, RAM usage, disk usage, network bandwidth, token consumption, financial cost, execution time, string similarity, and semantic similarity."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs. Accuracy is measured automatically via string and semantic similarity against LoCoMo ground truth answers. Given the subjective nature of Q&A quality, human evaluation would be relevant."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Not a machine learning training study. The benchmark questions serve as the evaluation set directly; there is no model training or tuning involved."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by experiment type (unconstrained/constrained), phase (loading/Q&A), memory framework, and cloud vs. edge in Tables 1-4. Response distributions (correct/wrong/IDK) are shown per condition."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 discusses the high IDK rate as a failure mode. The limitations note 'the DMAS responded to a significant number of questions with IDK' and suggest investigating root causes. Appendix A.3 shows an example of retrieved memories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that network constraints had 'limited impact on the overall DMAS performance' (less than 1% cost variation), which was contrary to what might be expected. The high IDK rate (58-68%) is also a negative result reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about mem0 outperforming Graphiti in efficiency and accuracy differences not being statistically significant are supported by Tables 1-6 and the z-test results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a factorial experimental design with two independent variables (memory framework, network profile) and controlled conditions. Claims like 'mem0 is the Pareto-optimal solution' follow from the controlled comparison. The design adequately supports the comparative claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on a single conversation from LoCoMo with a single LLM (gpt-4o-mini) and single SLM (qwen2.5:3b), but makes broad claims about 'DMAS memory' and 'the optimal choice' without adequately bounding to the specific models, benchmark, and conversation tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the results. For example, the high IDK rate could be due to the SLM's tool-calling quality, the responder prompt design, or memory retrieval quality — none of these are explored as confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'gpt-4o-mini' without a snapshot date or API version, and 'qwen2.5:3b-instruct-q4_K_M' which is a specific quantization but from an 'Ollama' image tagged 'latest'. No API version or snapshot date for gpt-4o-mini."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.2 provides the full system and user prompts for both the coordinator and responder agents, including the exact text with placeholders and their fill values explained."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of temperature, top-p, max tokens, or other sampling parameters for gpt-4o-mini or the Ollama-hosted SLM. These significantly affect output quality."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the multi-agent architecture in detail: coordinator with tool-calling SLM, memory agent with mem0/Graphiti backends, responder with LLM, proxy routing, and the loading/Q&A workflow. Figure 1 provides the system architecture diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how LoCoMo conversations are decomposed into sessions and turns, loaded via the coordinator's /remember endpoint. Appendix A.1 shows the JSON structure. The similarity scoring method is described in Section 2.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 'Limitations and future work' is a dedicated section discussing specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 identifies specific threats: the DMAS had 'little autonomy over its own actions,' the network constraints had 'limited impact,' and the high IDK rate needs investigation. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section discusses what future work should do but does not explicitly state what the current results do NOT show. For example, it does not state that results are limited to one conversation, one LLM, or one benchmark, despite these being significant scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'experimental results as well as notebooks for analyzing and evaluating the results are available' at the GitHub repository. Results are stored in CSV files."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes how data was collected: ten dependent variables measured during loading and Q&A phases, using Prometheus for metrics, telegraf for CPU/RAM/disk/network, and the analysis Jupyter notebook for Q&A results."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses the LoCoMo benchmark dataset, which is a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents the full pipeline: LoCoMo loaded via API → coordinator decomposes into turns → sent to memory agent → Q&A phase collects answers → analysis notebook stores CSV files. Monitoring via Prometheus and telegraf is also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section found in the paper. Authors are from KTH Royal Institute of Technology but no funding source is disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with KTH Royal Institute of Technology are clearly stated. Neither author appears affiliated with mem0 or Neo4j/Graphiti."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information disclosed, so independence cannot be assessed. The paper does mention mem0's $24M funding and Neo4j's $325M funding as context, but does not disclose any relationship."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement found in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-4o-mini to answer questions about LoCoMo conversations but does not state the model's training data cutoff. LoCoMo could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether gpt-4o-mini may have seen LoCoMo conversations or questions during training. The responder is prompted to use only provided memories, but contamination could still affect behavior."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LoCoMo was published in 2024. gpt-4o-mini was likely trained on data that could include LoCoMo. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports token consumption and USD cost per experiment/phase/memory. Section 4.1 converts computational cost to AWS Fargate pricing. Cost reporting is a central contribution of the paper."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports CPU (minutes), RAM (MB), disk (MB), and network (MB) usage per experiment. Table 3 reports total execution time. Section 4.1 converts these to financial cost using AWS pricing."
       }
     }
   },
   "claims": [
     {
       "claim": "mem0 significantly outperforms Graphiti in efficiency, with faster loading times, lower resource consumption, and minimal network overhead.",
       "evidence": "Tables 1-3: mem0 was 86.5% faster loading (unconstrained), used 104-232% less CPU, 38-8% less RAM, and orders of magnitude less network bandwidth than Graphiti.",
       "supported": "strong"
     },
     {
       "claim": "Accuracy differences between mem0 and Graphiti were not statistically significant.",
       "evidence": "Table 6: Two-proportion z-tests yield p=0.2269 (unconstrained) and p=0.4330 (constrained), both above α=0.05. Table 5 shows overlapping 95% Wilson CIs.",
       "supported": "strong"
     },
     {
       "claim": "mem0 is the Pareto-optimal solution balancing cost and accuracy in DMAS.",
       "evidence": "Section 4.2-4.3: Since accuracy difference is not significant and mem0 is substantially cheaper (40.2% lower cost unconstrained), mem0 dominates under the statistical Pareto framework.",
       "supported": "moderate"
     },
     {
       "claim": "Network constraints have limited impact on DMAS performance, with less than 1% cost variation.",
       "evidence": "Section 4.3 states 'computational and financial cost varies by less than 1%' with only 4.3% increased financial burden in constrained experiment.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This study compares mem0 (vector-based) and Graphiti (graph-based) long-term memory frameworks for distributed multi-agent systems using the LoCoMo benchmark. mem0 is substantially more efficient across all cost metrics (86.5% faster loading, 40.2% lower total cost) while accuracy differences are not statistically significant (p>0.05). Network constraints (200ms latency, 8 Mbits bandwidth) had minimal impact on performance. Both systems exhibited high IDK response rates (58-68%), suggesting fundamental limitations in memory retrieval quality.",
   "red_flags": [
     {
       "flag": "Single-run experiments",
       "detail": "All experiments appear to be single-run with no repetition across seeds or runs. No variance or standard deviation is reported. Results could be affected by API non-determinism (gpt-4o-mini) or system load variability."
     },
     {
       "flag": "Very low accuracy rates",
       "detail": "Both systems achieve only 6-11% accuracy on the LoCoMo benchmark, with 58-68% IDK responses. This suggests the experimental setup may have fundamental issues that limit the meaningfulness of accuracy comparisons."
     },
     {
       "flag": "Single conversation tested",
       "detail": "Only the first of ten LoCoMo conversations was used. Results may not generalize to other conversation types or memory workloads. N=199 questions from one conversation is a narrow evaluation."
     },
     {
       "flag": "Docker 'latest' tags for components",
       "detail": "Several critical components (Ollama, Prometheus, Qdrant, Socat, Jupyter Notebook) use 'latest' Docker tags (Table 7), making exact reproduction difficult as these tags change over time."
     }
   ],
   "cited_papers": [
     {
       "title": "Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory",
       "authors": ["P. Chhikara", "D. Khant", "S. Aryan", "T. Singh", "D. Yadav"],
       "year": 2025,
       "arxiv_id": "2504.19413",
       "relevance": "Core memory framework evaluated in this study; relevant to LLM agent memory and scalability."
     },
     {
       "title": "Evaluating Very Long-Term Conversational Memory of LLM Agents",
       "authors": ["A. Maharana", "D.-H. Lee", "S. Tulyakov", "M. Bansal", "F. Barbieri", "Y. Fang"],
       "year": 2024,
       "arxiv_id": "2402.17753",
       "relevance": "LoCoMo benchmark used in this study; foundational benchmark for evaluating LLM long-term memory."
     },
     {
       "title": "Large Language Model based Multi-Agents: A Survey of Progress and Challenges",
       "authors": ["T. Guo", "X. Chen", "Y. Wang", "R. Chang", "S. Pei", "N. V. Chawla", "O. Wiest", "X. Zhang"],
       "year": 2024,
       "arxiv_id": "2402.01680",
       "relevance": "Survey of LLM-based multi-agent systems covering progress and challenges."
     },
     {
       "title": "BudgetMLAgent: A Cost-Effective LLM Multi-Agent system for Automating Machine Learning Tasks",
       "authors": ["S. Gandhi", "M. Patwardhan", "L. Vig", "G. Shroff"],
       "year": 2025,
       "arxiv_id": "2411.07464",
       "relevance": "Addresses cost optimization in LLM multi-agent systems through profiling and efficient retrieval."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational work on cost-per-successful-query metric and LLM cost optimization."
     },
     {
       "title": "Green AI",
       "authors": ["R. Schwartz", "J. Dodge", "N. A. Smith", "O. Etzioni"],
       "year": 2020,
       "doi": "10.1145/3381831",
       "relevance": "Introduces accuracy-efficiency score framework referenced for balancing cost and accuracy in AI systems."
     },
     {
       "title": "Why Do Multi-Agent LLM Systems Fail?",
       "authors": ["M. Cemri", "M. Z. Pan", "S. Yang"],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Analyzes failure modes in multi-agent LLM systems including specification issues and inter-agent misalignment."
     },
     {
       "title": "Achilles Heel of Distributed Multi-Agent Systems",
       "authors": ["Y. Zhang", "Y. Li", "T. Zhao", "K. Zhu", "H. Wang", "N. Vasconcelos"],
       "year": 2025,
       "arxiv_id": "2504.07461",
       "relevance": "Identifies critical trustworthiness challenges in distributed multi-agent systems."
     },
     {
       "title": "Zep: A Temporal Knowledge Graph Architecture for Agent Memory",
       "authors": ["P. Rasmussen", "P. Paliychuk", "T. Beauvais", "J. Ryan", "D. Chalef"],
       "year": 2025,
       "arxiv_id": "2501.13956",
       "relevance": "Graph-based agent memory framework relevant to evaluating knowledge graph approaches for LLM agents."
     }
   ]
 }