ai-research-survey

scan.json (25641B)

---

 {
   "paper": {
     "title": "Evaluating Very Long-Term Conversational Memory of LLM Agents",
     "authors": ["Adyasha Maharana", "Dong-Ho Lee", "Sergey Tulyakov", "Mohit Bansal", "Francesco Barbieri", "Yuwei Fang"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2402.17753"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs with limited context struggle with very long conversations (best base model GPT-4-turbo scores 32.1 vs human 87.9 on QA). Long-context LLMs improve factual recall but catastrophically fail on adversarial questions (2.1% for GPT-3.5-turbo-16k vs 70.2% for GPT-4-turbo base). RAG with observations provides the best balance, particularly outperforming dialog-based and summary-based retrieval. Event summarization reveals that long-context models paradoxically underperform base models, suggesting they fail to utilize extended context appropriately.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Code and data to be available at https://snap-research.github.io/locomo' — a promise of future release, not a confirmed available release at time of publication."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as code — dataset is promised to be available at the project page but described as future release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'Nvidia A6000 server with FP32' and specific APIs but provides no requirements.txt, Dockerfile, or detailed dependency listing."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Implementation details are scattered across the paper and appendix but no consolidated reproduction guide exists."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, and 4 are reported as point estimates (F1 scores, ROUGE scores, FactScores) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('outperforms', 'improvement') but reports no statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with baselines, e.g., 'improvements ranging from 22-66%', 'performance that is 83% lower than the base model', 'lags behind human levels (by 56%)'. Percentage improvements with baselines provide effect size context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains only 50 conversations and 7,512 QA pairs. No justification for why 50 conversations is sufficient, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section C.2 states 'We report results from a single inference run for each model in our experiments.' No variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares base LLMs, long-context LLMs, and RAG approaches, plus human performance as an upper bound. Multiple models serve as baselines (Mistral-7B, Llama-2-Chat-70B, GPT-3.5-turbo, GPT-4-turbo)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-4-turbo, GPT-3.5-turbo-16k, Llama-2-Chat-70B, and Mistral-7B, all contemporary at the time of writing (January 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The RAG experiments ablate retrieval units (dialog, observation, summary) and top-k values. The multimodal dialog generation task ablates training variants (base, +summary, +observation)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "QA uses F1 and retrieval recall@k. Event summarization uses ROUGE-1/2/L and FactScore (precision, recall, F1). Multimodal dialog uses BLEU, ROUGE-L, and MMRelevance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Human performance is measured as an upper bound on the QA task (Table 2, 87.9 overall F1). Human annotators also verified and edited the dataset (Section 3.4)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No explicit train/test/dev split is described for the 50 LOCOMO conversations used for evaluation. The 50 MiniGPT-5 training conversations are separately generated, but the evaluation benchmark itself has no described held-out split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "QA results are broken down by 5 reasoning types (single-hop, multi-hop, temporal, open-domain, adversarial) in Tables 2 and 3. RAG results are broken down by retrieval unit and top-k."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 presents a taxonomy of 5 error categories in event summarization (missing information, hallucination, misunderstanding dialog cues, speaker attribution errors, saliency errors) with examples in Table 7."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: long-context LLMs catastrophically fail on adversarial questions (2.1%), long-context models underperform base models on event summarization, summary-based RAG does not significantly improve performance despite high recall."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about LLM challenges with lengthy conversations, improvements from long-context/RAG, and lagging behind human performance are all supported by Tables 2-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'Employing strategies like long-context LLMs or RAG can offer improvements' and 'introducing improper context from inaccurate retrieval can lead to a decline in performance' without controlled causal designs — these are observational comparisons across different model configurations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'LLM Agents' generally but tests only a few specific models (GPT-3.5/4, Llama-2, Mistral) on a single synthetic dataset of 50 conversations. The paper does not bound its claims to these specific models and dataset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the adversarial question collapse in long-context models could be due to prompt formatting rather than context length, but this is not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures F1 on QA and FactScore on summarization as proxies for 'long-term memory' and 'comprehension' without discussing the gap between these automated metrics and actual memory capabilities."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to as 'gpt-3.5-turbo', 'gpt-4-turbo', 'Mistral-7B', 'Llama-2-Chat-70B' without specific version snapshots or dates. The paper states 'as of January 2024' but does not give exact API version strings."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Extensive prompts are provided in the appendix: persona generation (Fig. 5), event graph generation (Fig. 6), summary generation (Fig. 8), observation extraction (Fig. 9), and image sharing/reaction (Fig. 10)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section C.2: 'temperature set to 0 and topp set to 1 for evaluation.' MiniGPT-5 trained for 10 epochs using default hyperparameters from original codebase."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The generative agent architecture is described in detail in Section 3.3: reflect & respond mechanism with short/long-term memory, observation extraction, summary generation, and image sharing/reaction functions. The RAG pipeline (DRAGON retriever + reader) is also described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from persona selection → event graph generation → dialog generation → human verification/editing is documented across Sections 3.1-3.4 with details in Appendix A. Annotators edited ~15% of turns and removed/substituted ~19% of images."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations' provides substantive discussion across 5 sub-topics: hybrid data, limited multimodal behavior, language limitation, closed-source LLMs, and evaluation challenges."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific: 'this dataset may not fully reflect the nuances of real-world online conversations', 'images in our dataset can be replaced with their captions without much loss of information', 'LLMs are prone to generating verbose answers' creating evaluation challenges."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the pipeline works only for English, acknowledges synthetic data limitations, and notes that web-sourced images lack visual long-term consistency of personal photos. Section 9 explicitly states 'We do not make any recommendations for real-world policies based on this study.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is promised for future release but not available at publication time."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire data collection pipeline is described in detail: persona generation from MSC dataset, event graph construction, LLM agent dialog generation, human verification/editing (Sections 3.1-3.4 and Appendix A)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The human annotators are described only as 'in-house annotators' (Section B.3). No details on how they were selected, their qualifications, or demographics — 'we were unable to obtain their demographics due to the confidential nature of such information.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is fully documented: persona selection → expansion via GPT-3.5 → event graph generation via text-davinci-003 → agent dialog generation → human filtering/editing. Statistics on edit rates provided (15% dialog turns, 19% images)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UNC Chapel Hill, USC, and Snap Inc."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three authors are from Snap Inc., which has commercial interest in conversational AI agents. This potential conflict is not discussed. No funding disclosure makes independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models (GPT-3.5-turbo, GPT-4-turbo, Mistral-7B, Llama-2-Chat-70B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The LOCOMO benchmark is newly created, which reduces contamination risk, but this is not explicitly discussed. The MSC personas used as seeds are public and could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the generated conversations or their source personas could overlap with model training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. Human annotators were used for dataset construction, not as study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Dataset annotation is not a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or per-example costs reported despite extensive use of OpenAI APIs for both data generation and evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "MiniGPT-5 training took 'approximately 30 hours on a single A6000 GPU' (Section C.2). However, no total API costs for data generation or LLM evaluation are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section C.2 explicitly states 'We report results from a single inference run for each model.' No seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section C.2 states 'We report results from a single inference run for each model in our experiments.' The number is stated, though it is only 1."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described for evaluation. MiniGPT-5 uses 'default hyperparameters from original codebase' with no search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how best configurations were selected across the various RAG top-k settings or model choices."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons across models, retrieval strategies, and question types with no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the LOCOMO benchmark and evaluate models on it without acknowledging potential bias from evaluating on their own benchmark."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes (Mistral-7B vs GPT-4-turbo) are compared without discussing compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether QA F1, FactScore, and MMRelevance actually measure 'long-term memory' as claimed, or whether the synthetic benchmark reflects real conversation dynamics."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "RAG models use DRAGON retriever + GPT-3.5-turbo-16k reader, confounding retrieval quality with reader capability. Base models have different context truncation strategies. These scaffolding differences are not addressed as confounds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The LOCOMO conversations are generated from MSC personas which are publicly available. No discussion of whether these personas or similar conversation patterns appeared in model training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, e.g., whether QA question phrasing provides hints."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All 50 conversations are generated by gpt-3.5-turbo, and GPT-3.5-turbo is also used as a baseline model. The evaluated model generated the test data it is being tested on, which is a significant non-independence issue not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are employed."
       }
     }
   },
   "claims": [
     {
       "claim": "Long-context LLMs and RAG improve QA memory capabilities by 22-66% but still lag behind human performance by 56%.",
       "evidence": "Table 2: GPT-3.5-turbo-16k at 16K context achieves 37.8 overall F1 vs human 87.9. Base GPT-3.5-turbo at 22.4. Improvement ranges from 22-66% depending on context size (Section 6.1).",
       "supported": "strong"
     },
     {
       "claim": "Long-context LLMs are prone to hallucinations on adversarial questions, with performance dropping to 2.1% compared to 70.2% for GPT-4-turbo base.",
       "evidence": "Table 2: GPT-3.5-turbo-16k at 16K context scores 2.1% on adversarial questions vs GPT-4-turbo base at 70.2%.",
       "supported": "strong"
     },
     {
       "claim": "RAG is most effective when conversations are stored as observations rather than raw dialog or summaries.",
       "evidence": "Table 3: Observation-based RAG achieves 41.4 overall F1 at top-5 vs dialog-based 31.7 and summary-based 29.9.",
       "supported": "moderate"
     },
     {
       "claim": "Long-context models show poor performance on event summarization, lagging behind the base model by 14%.",
       "evidence": "Table 4: GPT-3.5-turbo achieves 45.9 FactScore F1 vs GPT-3.5-turbo-16k at 39.9.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single-run results",
       "detail": "All experiments are single inference runs with no variance reporting. Results could vary substantially across runs, especially for temperature=0 API calls which may still have non-determinism."
     },
     {
       "flag": "Evaluated model generated test data",
       "detail": "GPT-3.5-turbo was used to generate the LOCOMO conversations and is also used as a baseline model. This creates a fundamental non-independence issue — the model may perform differently on data it generated vs. truly novel conversations."
     },
     {
       "flag": "Very small dataset",
       "detail": "Only 50 conversations form the entire benchmark. No justification for why this sample size is sufficient for the claims made."
     },
     {
       "flag": "No statistical tests",
       "detail": "All comparative claims are based on raw number comparisons with no significance testing. Given the small dataset (50 conversations), differences could easily be due to chance."
     },
     {
       "flag": "Company conflict not discussed",
       "detail": "Three of six authors are from Snap Inc., which has commercial interest in conversational AI. No competing interests disclosure."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["Joon Sung Park", "Joseph O'Brien", "Carrie Jun Cai", "Meredith Ringel Morris", "Percy Liang", "Michael S. Bernstein"],
       "year": 2023,
       "relevance": "Foundation architecture for LLM-based agents with memory and reflection, directly used in LOCOMO's generative pipeline."
     },
     {
       "title": "Beyond goldfish memory: Long-term open-domain conversation",
       "authors": ["Jing Xu", "Arthur Szlam", "Jason Weston"],
       "year": 2022,
       "relevance": "MSC dataset and prior work on long-term dialogue that LOCOMO extends by 9x in conversation length."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "authors": ["Nelson F. Liu", "Kevin Lin", "John Hewitt", "Ashwin Paranjape", "Michele Bevilacqua", "Fabio Petroni", "Percy Liang"],
       "year": 2024,
       "relevance": "Documents LLM challenges with long context utilization, directly relevant to LOCOMO's findings on long-context model failures."
     },
     {
       "title": "FActScore: Fine-grained atomic evaluation of factual precision in long form text generation",
       "authors": ["Sewon Min", "Kalpesh Krishna", "Xinxi Lyu"],
       "year": 2023,
       "relevance": "Evaluation metric used for event summarization task in LOCOMO benchmark."
     },
     {
       "title": "Retrieval augmentation reduces hallucination in conversation",
       "authors": ["Kurt Shuster", "Spencer Poff", "Moya Chen", "Douwe Kiela", "Jason Weston"],
       "year": 2021,
       "relevance": "Foundational work on RAG for dialogue, directly relevant to LOCOMO's RAG evaluation."
     },
     {
       "title": "Conversation Chronicles: Towards diverse temporal and relational dynamics in multi-session conversations",
       "authors": ["Jihyoung Jang", "Minseong Boo", "Hyounghun Kim"],
       "year": 2023,
       "relevance": "Most comparable prior work on multi-session dialogues, limited to 5 sessions vs LOCOMO's 35."
     },
     {
       "title": "Replug: Retrieval-augmented black-box language models",
       "authors": ["Weijia Shi", "Sewon Min", "Michihiro Yasunaga"],
       "year": 2023,
       "arxiv_id": "2301.12652",
       "relevance": "RAG technique for augmenting LLMs, relevant to retrieval-based approaches evaluated in LOCOMO."
     },
     {
       "title": "MemoryBank: Enhancing large language models with long-term memory",
       "authors": ["Wanjun Zhong", "Lianghong Guo", "Qiqi Gao", "Yanlin Wang"],
       "year": 2023,
       "arxiv_id": "2305.10250",
       "relevance": "LLM memory augmentation approach directly relevant to the long-term memory evaluation problem LOCOMO addresses."
     },
     {
       "title": "Booookscore: A systematic exploration of book-length summarization in the era of LLMs",
       "authors": ["Yapei Chang", "Kyle Lo", "Tanya Goyal", "Mohit Iyyer"],
       "year": 2023,
       "relevance": "Long-form summarization evaluation methodology relevant to LOCOMO's event summarization task."
     },
     {
       "title": "How to train your dragon: Diverse augmentation towards generalizable dense retrieval",
       "authors": ["Sheng-Chieh Lin", "Akari Asai", "Minghan Li"],
       "year": 2023,
       "relevance": "DRAGON retrieval model used in LOCOMO's RAG experiments."
     }
   ]
 }