ai-research-survey

scan.json (31264B)

---

 {
   "paper": {
     "title": "Episodic Memories Generation and Evaluation Benchmark for Large Language Models",
     "authors": ["Alexis Huet", "Zied Ben Houidi", "Dario Rossi"],
     "year": 2025,
     "venue": "International Conference on Learning Representations",
     "arxiv_id": "2501.13121",
     "doi": "10.48550/arXiv.2501.13121"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Even state-of-the-art LLMs (GPT-4o, Claude 3.5 Sonnet, o1-mini, Llama 3.1 405B) struggle with episodic memory tasks on a synthetic contamination-free benchmark, particularly when questions require recalling multiple related events (F1 ≤0.60 for 2+ events). Performance degrades consistently with cue overload, and naive fine-tuning fails to generalize beyond single-event memorization. RAG generally outperforms in-context memory (except for GPT-4o), and chronological ordering and entity state tracking are especially challenging (≤36% latest state accuracy, ≤18% full set recall).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository is provided: 'Code and data available at Huet et al. (2025)' with URL https://github.com/ahstat/episodic-memory-benchmark (footnote 1, reference section)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "11 datasets are released (Tab. 28) through the GitHub repository. The paper states 'release open source code and datasets' and footnote 2 references the list in Tab. 28."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No mention of requirements.txt, Dockerfile, conda environment, or dependency specifications in the paper. Only the models used are named without library/environment setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark generation process is described in extensive detail (Appendix B), the paper does not provide step-by-step reproduction instructions with specific commands. The methodology is described algorithmically but not as executable steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported throughout, e.g., Tab. 3: '0.84±0.37', '0.81±0.38'. All results tables include ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Wilcoxon signed-rank test with Holm's method adjustment is used for model comparisons (Fig. 3, Fig. 7). Mann-Whitney U tests are used in ablation studies (Tab. 21, 23, 25)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "F1 scores are reported with full context: baseline comparisons show magnitude of differences (e.g., fine-tuned model drops from F1=0.83 for 1 event to F1=0.19 for 6+ events in Tab. 3). Performance differences between conditions are quantified absolutely."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The question selection targets Ntarget=5 per bin per question id (Appendix B.2.4) but no formal power analysis or justification for why 456/686 questions are sufficient for the statistical conclusions drawn."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported across questions for all experimental conditions in Tab. 3, 13, 14, 16-20, 22, 24, 27."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: six LLMs across three memory strategies (in-context, RAG, fine-tuning). Section 5.1 describes the baseline setup."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, GPT-4o-mini, Claude 3.5 Sonnet, Llama 3.1 405B, and o1-mini — all state-of-the-art at time of submission (late 2024/early 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies in Appendix E: book size (E.1), RAG granularity (E.2), number of cues and traces (E.3, E.4), book generation model (E.5), chapter ordering (E.6), and event plausibility (E.7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "F1-score is the primary metric (Section 4.3), and Kendall's τ coefficient is additionally used for chronological ordering questions (Section 4.3, Tab. 4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is performed. Evaluation uses an LLM-as-a-judge approach (Section 4.3, Appendix B.3). Manual analysis of GPT-4o failures in Appendix E.8 covers only 24 questions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "For the fine-tuning experiment, 'all questions involving a single chapter (i.e., corresponding to the bin {1}) are present in both the training and the test sets' (Appendix B.2.5). Train/test contamination exists in the fine-tuning condition."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by number of matching events (Tab. 3), cue type (Fig. 4, Tab. 16-17), retrieval type (Tab. 18), and detailed (cue, retrieval) pairs (Tab. 19)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.8 provides manual analysis of 24 failed GPT-4o responses on 0-event questions, categorizing failures into 'inner' (17 cases) and 'outer' (7 cases) with detailed examples of confabulation patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning is shown to fail at generalization (F1=0.00 on 0-event questions, drops from 0.83 to 0.19 as events increase). o1-mini performs worst at recall despite best confabulation avoidance. Claude 3.5 Sonnet in-context has a known bug (Appendix F)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims that 'even the most advanced LLMs struggle with episodic memory tasks, particularly when dealing with multiple related events' are directly supported by Tab. 3 (F1 ≤0.60 for 2+ events) and Fig. 3 (model rankings with statistical tests)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are generally supported through controlled comparisons. 'RAG generally outperforms in-context counterparts' is supported by comparing the same model across memory strategies (Fig. 3). Ablation studies (Appendix E) systematically vary single factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested models and benchmark. The paper specifies which models were evaluated, acknowledges limitations including 'limited domain scope' (Section 6), and explicitly notes the benchmark covers 'human-like protagonists within fictional contexts.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple alternative explanations are investigated: Claude vs. GPT book generation bias (Appendix E.5), chronological vs. unordered presentation (E.6), realistic vs. non-realistic events (E.7), and paragraph vs. chapter RAG granularity (E.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly draws parallels between their benchmark and established human episodic memory tests (AMI, Autobiographical Interview in Section 2 and Appendix A.3.1), and clearly defines what F1 score on cue-based recall measures. Section 3 articulates the cognitive science framework underlying the measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Book generation models include version dates: 'Claude 3.5 Sonnet (2024-06-20)' and 'GPT-4o (2024-05-13)' (Appendix B.1.5). However, evaluation models are listed without version/snapshot dates: just 'GPT-4o', 'GPT-4o-mini', 'Claude 3 Haiku', 'Claude 3.5 Sonnet', 'o1-mini' (Section 5.1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in the appendix: chapter generation (Listing 7), chapter verification (Listing 8), evaluation (Listings 12, 14), answer generation for in-context (Listing 15), RAG (Listing 16), and fine-tuning (Listing 17)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning hyperparameters are reported (30 epochs, batch size 64, LR multiplier 1.8; footnote 5) and RAG uses 'text-embedding-3-small.' However, inference hyperparameters (temperature, top-p, max tokens) for the evaluated models are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct prompting of LLMs, RAG retrieval, or fine-tuning — none of which involve agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data generation pipeline is extensively documented in Appendix B: universe creation (B.1.1-B.1.2), event generation with geometric sampling (B.1.3), meta-data generation (B.1.4), chapter generation and verification (B.1.5-B.1.7), question selection with filtering counts (B.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Summary and Limitations' provides dedicated subsections addressing temporal representation, event independence, limited domain scope, and training limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are identified: 'Our benchmark relies on explicit temporal markers, which may not fully capture the nuanced ways time is expressed in natural language'; 'The independent generation of chapters... does not capture the interconnected and causal nature of real-world events' (Section 6)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states specific scope boundaries: benchmark is limited to explicit temporal markers (not implicit references like 'yesterday'), events are independently generated (no causal chains), and domain is restricted to 'human-like protagonists within fictional contexts.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code and all 11 benchmark datasets are released on GitHub (https://github.com/ahstat/episodic-memory-benchmark). Tab. 28 lists all produced benchmarks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic data generation procedure is described in detail in Appendix B: universe components (B.1.1), event generation with truncated geometric distribution (B.1.3), chapter generation with LLM (B.1.5), and multi-stage verification (B.1.6-B.1.7)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is entirely synthetic, generated procedurally from defined universe components."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with counts at each stage: 200 events generated → filtering for (t,ent)/(t,s) conflicts → itermax=10 chapter generation attempts → direct verification → LLM-as-a-judge verification → 196 validated chapters (Tab. 7). Question pipeline: 7056 raw questions → 3886 after dedup → 686 after selection (Appendix B.2)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure, acknowledgments section, or grant information is present in the paper. Authors are listed as Huawei Technologies employees but no explicit funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is clearly stated: 'Huawei Technologies Co., Ltd., Paris, France' on the title page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The implicit funder (Huawei) does not have a direct financial interest in the specific outcomes — the paper evaluates third-party models (GPT-4o, Claude, Llama, o1-mini), none of which are Huawei products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-4o, Claude 3.5 Sonnet, Llama 3.1, o1-mini). While the benchmark is designed to be contamination-free, the models' training cutoffs are not mentioned."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination as a design requirement: 'We synthesize a unique episodic memory benchmark, free from contamination' (Section 1). Section 2 identifies data leakage as a limitation of existing benchmarks. The synthetic generation approach structurally prevents train/test overlap."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is specifically designed to be contamination-free: 'avoid data leakage, ensuring that the evaluation is free from contamination' (Section 2). The synthetic generation approach guarantees benchmark data did not exist before model training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The benchmark evaluates LLMs on synthetic tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM performance on synthetic benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or inference latency are reported for the evaluation runs across six models and three memory strategies."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours, API spend, or total compute for benchmark generation or model evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single evaluation runs. No seed sensitivity analysis is reported for model inference. Benchmark generation uses a fixed seed (Appendix B.1.2) but the paper does not vary it for evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per model is not explicitly stated. Results appear to be single-run per model-memory configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For fine-tuning, hyperparameters (30 epochs, batch size 64, LR multiplier 1.8) are stated but no search budget is described. RAG top-K is set to 'top-K paragraphs' without discussing how K was chosen. No search budget for any configuration."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Fine-tuning hyperparameters and RAG parameters appear chosen without explanation of the selection process or reporting of alternative configurations tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Holm's method is used for adjusting Wilcoxon signed-rank tests in the Critical Difference plots (Fig. 3, Fig. 7). This is explicitly stated: 'Wilcoxon signed-rank test... adjusted by Holm's method.'"
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate third-party models on their own benchmark but do not discuss potential benchmark design bias that could favor or disadvantage certain model architectures."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Performance is not reported as a function of compute budget. Different models and memory strategies have different compute requirements (e.g., RAG adds embedding overhead, fine-tuning requires training) but this is not quantified or compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses what the benchmark measures and its relationship to cognitive science episodic memory constructs (Section 3, Appendix A). It draws explicit parallels to human episodic memory tests (AMI, Autobiographical Interview) and explains how cue-based recall maps to LLM prompting."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated via direct prompting, RAG, or fine-tuning without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is synthetically generated specifically to avoid temporal leakage: 'free from contamination' by design. The synthetic data was generated after model training, making prior exposure impossible."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of whether the evaluation setup could leak answer information through context. For in-context learning, the full book is provided (by design), but the paper does not discuss whether question formulations could inadvertently signal answers."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Questions are generated from a shared universe with overlapping elements (same entities, locations, dates across chapters), but the paper does not discuss whether this shared structure could create dependencies between test questions that affect evaluation validity."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic benchmark generation itself serves as a concrete leakage prevention method — all data is generated de novo, making it structurally impossible for models to have seen it during training. This is highlighted as a key design feature in Sections 1-2."
       }
     }
   },
   "claims": [
     {
       "claim": "Even the most advanced LLMs struggle with episodic memory tasks, particularly with multiple related events or complex spatio-temporal relationships, even in contexts as short as 10k-100k tokens.",
       "evidence": "Tab. 3 shows F1 scores ≤0.60 for all models when questions involve 2+ matching events on the long book. Tab. 13 shows suboptimal performance even on the short book (10k tokens). Fig. 3 ranks all models with statistical tests.",
       "supported": "strong"
     },
     {
       "claim": "RAG generally outperforms in-context memory for episodic tasks, with the exception of GPT-4o.",
       "evidence": "Fig. 3 shows RAG variants generally rank higher than in-context counterparts. GPT-4o in-context achieves the highest average rank, statistically tied with Claude 3.5 Sonnet RAG.",
       "supported": "moderate"
     },
     {
       "claim": "Performance degrades consistently as the number of events matching a cue increases (cue overload effect).",
       "evidence": "Tab. 3 shows monotonic F1 decline across all models from 1-event to 6+ events (e.g., GPT-4o: 0.81→0.60→0.57→0.53). Fig. 4 heatmaps visualize this gradient.",
       "supported": "strong"
     },
     {
       "claim": "Naive fine-tuning fails to achieve deep understanding of episodic events and merely overfits to single learned facts.",
       "evidence": "Tab. 3: fine-tuned GPT-4o-mini achieves F1=0.83 for 1-event questions but drops to 0.37 for 2 events and 0.19 for 6+ events. It scores F1=0.00 on 0-event questions (always hallucinates), confirming it memorized specific answers without generalizing.",
       "supported": "strong"
     },
     {
       "claim": "o1-mini performs best at avoiding confabulation but worst at actual event recall.",
       "evidence": "Tab. 3: o1-mini achieves F1=0.97 on 0-event questions (confabulation avoidance) but only 0.05 on 1-event questions on the long book. On the short book (Tab. 13), o1-mini performs significantly better and is statistically equivalent to GPT-4o (Fig. 7).",
       "supported": "strong"
     },
     {
       "claim": "Performance varies by cue type, with content cues being easiest and time cues being hardest.",
       "evidence": "Fig. 4 shows a gradient from content (top) to time (bottom) across models, with content cues consistently yielding higher F1 scores than time cues.",
       "supported": "moderate"
     },
     {
       "claim": "Models have significant difficulty with chronological ordering and entity state tracking, especially for multi-event questions.",
       "evidence": "Tab. 4: ≤36% accuracy for latest state recall, ≤18% for full set recall across all models. Kendall's τ is low even for exact matches, showing ordering failures.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Unspecified judge model in LLM-as-a-judge evaluation",
       "detail": "The paper uses an 'LLM-as-a-judge approach' (Section 4.3) for evaluation but does not explicitly state which model serves as the judge. Since 4-8% of scores are partial matches requiring judgment, the choice of judge model could affect results."
     },
     {
       "flag": "No multi-run evaluation or seed sensitivity",
       "detail": "Results appear to be from single evaluation runs per model-memory configuration. LLM outputs can vary across runs due to sampling, but no variance across runs is reported. The ± values in tables represent variance across questions, not across repeated runs."
     },
     {
       "flag": "Missing inference hyperparameters",
       "detail": "Temperature, top-p, and max tokens are not reported for any evaluated model. These parameters significantly affect LLM output, especially for memory recall tasks where more deterministic sampling (low temperature) might improve results."
     }
   ],
   "cited_papers": [
     {
       "title": "Human-like episodic memory for infinite context llms",
       "authors": ["Zafeirios Fountas", "Martin A Benfeghoul", "Adnan Oomerjee", "Fenia Christopoulou", "Gerasimos Lampouras", "Haitham Bou-Ammar", "Jun Wang"],
       "year": 2024,
       "arxiv_id": "2407.09450",
       "relevance": "Proposes human-like episodic memory mechanisms for LLMs, directly relevant to extending LLM memory capabilities."
     },
     {
       "title": "Larimar: Large language models with episodic memory control",
       "authors": ["Payel Das", "Subhajit Chaudhury", "Elliot Nelson", "Igor Melnyk", "Sarath Swaminathan", "Sihui Dai", "Aurélie Lozano", "Georgios Kollias", "Vijil Chenthamarakshan", "Soham Dan"],
       "year": 2024,
       "arxiv_id": "2403.11901",
       "relevance": "Proposes episodic memory control architecture for LLMs, directly addressing the memory limitations this benchmark evaluates."
     },
     {
       "title": "Michelangelo: Long context evaluations beyond haystacks via latent structure queries",
       "authors": ["Kiran Vodrahalli", "Santiago Ontanon", "Nilesh Tripuraneni", "Kelvin Xu", "Sanil Jain", "Rakesh Shivanna", "Jeffrey Hui", "Nishanth Dikkala", "Mehran Kazemi", "Bahare Fatemi"],
       "year": 2024,
       "arxiv_id": "2409.12640",
       "relevance": "Introduces latent structure queries for long-context evaluation, the closest prior work to this episodic memory benchmark."
     },
     {
       "title": "Babilong: Testing the limits of llms with long context reasoning-in-a-haystack",
       "authors": ["Yuri Kuratov", "Aydar Bulatov", "Petr Anokhin", "Ivan Rodkin", "Dmitry Sorokin", "Artyom Sorokin", "Mikhail Burtsev"],
       "year": 2024,
       "arxiv_id": "2406.10149",
       "relevance": "Long-context reasoning benchmark extending bAbI tasks, which this paper positions as lacking episodic memory assessment."
     },
     {
       "title": "Ruler: What's the real context size of your long-context language models?",
       "authors": ["Cheng-Ping Hsieh", "Simeng Sun", "Samuel Kriman", "Shantanu Acharya", "Dima Rekesh", "Fei Jia", "Boris Ginsburg"],
       "year": 2024,
       "arxiv_id": "2404.06654",
       "relevance": "Evaluates actual context utilization of long-context LLMs, directly relevant to understanding context window limitations."
     },
     {
       "title": "Gemini 1.5: Unlocking multimodal understanding across millions of tokens of context",
       "authors": ["Machel Reid", "Nikolay Savinov", "Denis Teplyashin", "Dmitry Lepikhin", "Timothy Lillicrap", "Jean-baptiste Alayrac", "Radu Soricut", "Angeliki Lazaridou", "Orhan Firat", "Julian Schrittwieser"],
       "year": 2024,
       "arxiv_id": "2403.05530",
       "relevance": "Demonstrates multi-million token context capabilities and includes needle-in-haystack evaluation that this paper critiques as insufficient."
     },
     {
       "title": "Extracting training data from large language models",
       "authors": ["Nicholas Carlini", "Florian Tramer", "Eric Wallace", "Matthew Jagielski", "Ariel Herbert-Voss", "Katherine Lee", "Adam Roberts", "Tom Brown", "Dawn Song", "Ulfar Erlingsson"],
       "year": 2021,
       "relevance": "Demonstrates LLM memorization of training data, relevant to understanding model memory and data contamination risks."
     },
     {
       "title": "Quantifying memorization across neural language models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski", "Katherine Lee", "Florian Tramer", "Chiyuan Zhang"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Quantifies memorization as function of model scale and context length, directly informing the episodic memory benchmark design."
     },
     {
       "title": "Augmenting language models with long-term memory",
       "authors": ["Weizhi Wang", "Li Dong", "Hao Cheng", "Xiaodong Liu", "Xifeng Yan", "Jianfeng Gao", "Furu Wei"],
       "year": 2023,
       "arxiv_id": "2306.07174",
       "relevance": "Proposes LongMem architecture for augmenting LLMs with persistent memory, relevant to addressing the episodic memory limitations documented in this benchmark."
     },
     {
       "title": "Many-shot in-context learning",
       "authors": ["Rishabh Agarwal", "Avi Singh", "Lei M Zhang", "Bernd Bohnet", "Stephanie Chan", "Ankesh Anand", "Zaheer Abbas", "Azade Nova", "John D Co-Reyes", "Eric Chu"],
       "year": 2024,
       "arxiv_id": "2404.11018",
       "relevance": "Studies in-context learning scaling with many examples, relevant to understanding LLM ability to process and retain information in context."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Benchmark framework and code are released for researchers, but the benchmark itself is a research tool not directly applicable to practitioner workflows."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "LLMs struggling with memory tasks is not hugely surprising, though the degree of failure at even 10k-token contexts and the complete failure of fine-tuning to generalize is somewhat unexpected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; straightforward benchmark evaluation paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository with code and 11 datasets is released, allowing researchers to run the benchmark on their own models."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From Huawei (known but not top-tier AI lab); evaluates well-known models (GPT-4, Claude) which adds some recognition."
     }
   }
 }