ai-research-survey

scan.json (31827B)

---

 {
   "paper": {
     "title": "Forgetful but Faithful: A Cognitive Memory Architecture and Benchmark for Privacy-Aware Generative Agents",
     "authors": ["Saad Alqithami"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.12856",
     "doi": "10.48550/arXiv.2512.12856"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "The paper proposes MaRS, a typed memory architecture for generative agents with provenance-tracked nodes and forgetting policies, plus FiFA, a benchmark measuring coherence, goal completion, social recall, privacy, and cost. Across 300 simulation runs (6 policies × 5 budgets × 10 seeds), the naive Random Drop policy surprisingly achieves the highest composite score (0.635), while the sophisticated Hybrid policy scores lower (0.589) due to compute overhead. The abstract claims Hybrid achieves ≈0.911 composite, directly contradicting the reported results. Policy choice dominates budget size as the primary performance lever across the 2K–32K token range.",
   "claims": [
     {
       "claim": "The Hybrid policy delivers the best composite performance (≈0.911) while maintaining tractable cost and high privacy scores.",
       "evidence": "Abstract claim. Table 2 in §6.5.1 actually shows Random Drop at 0.635 as highest composite and Hybrid at 0.589 — a direct contradiction. No 0.911 figure appears anywhere in the results.",
       "supported": "unsupported"
     },
     {
       "claim": "Random Drop achieves the highest Composite score (0.635 ± 0.024) among the evaluated policies.",
       "evidence": "Table 2 (§6.5.1) with 95% confidence intervals. Statistical significance confirmed in Table 3 (F=5.93, p<0.001) and pairwise comparison in Table 4 (Random Drop vs. Hybrid: d=1.41, p<0.001).",
       "supported": "moderate"
     },
     {
       "claim": "Cost efficiency shows the largest policy effect (η² = 0.832), with simple policies dominating.",
       "evidence": "Table 3 (§6.5.2): F=86.43, p<0.0001, η²=0.832. FIFO (0.941), Random Drop (0.935) vs. Hybrid (0.730). Consistent with the theoretical complexity analysis.",
       "supported": "strong"
     },
     {
       "claim": "Budget independence holds over the 2K–32K token range: absolute scores improve modestly but policy rankings remain stable.",
       "evidence": "§6.7.1: omnibus F values in low single digits, p>0.27 across metrics. Figures 4 and 6 visualize the stable rankings. Consistent with the budget–utility Lipschitz bound.",
       "supported": "moderate"
     },
     {
       "claim": "Narrative coherence differences among policies are highly significant with large effect size (η² = 0.351).",
       "evidence": "Table 3 (§6.5.2): F=9.45, p<0.0001. Random Drop leads at 0.667 ± 0.074 vs. LRU at 0.501 ± 0.023 (Table 2).",
       "supported": "moderate"
     },
     {
       "claim": "MaRS supports (ε, δ)-differential privacy guarantees at the retention decision boundary.",
       "evidence": "Theorem A1 in Appendix C provides the proof via the exponential mechanism. However, no empirical validation of the DP guarantees is presented; the DP tie-break 'fires infrequently' in practice (§6.6.4).",
       "supported": "weak"
     },
     {
       "claim": "All goal completion rates are extremely low (<8%) across all policies and budgets.",
       "evidence": "Table 2: best is Random Drop at 0.078 ± 0.010. This is an empirical observation from the benchmark runs, but it raises questions about benchmark calibration.",
       "supported": "moderate"
     }
   ],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or any link to released code appears anywhere in the paper. The paper describes a complex framework (MaRS + FiFA) but provides no implementation artifacts."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No simulation data, scenario definitions, or experimental outputs are released. The benchmark results and simulation logs are not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, hardware descriptions, or software requirements are provided anywhere in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or runnable scripts are provided. The paper describes the architecture conceptually but provides no steps to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports 95% confidence intervals for all metrics (e.g., '0.635 ± 0.024'). §6.3.2 describes bootstrap confidence intervals, and §6.4.2 specifies cluster-robust sandwich estimators."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports ANOVA F-statistics and p-values for each metric. Table 4 reports Holm–Bonferroni adjusted p-values for pairwise comparisons. §6.4.2 describes Wilcoxon signed-rank tests and McNemar's test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports η² effect sizes for each metric. Table 4 reports Cohen's d with Hedges' correction (e.g., d=1.41 for Random Drop vs. Hybrid). §6.4.2 also mentions Cliff's δ."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The design uses 10 seeds, 5 budgets, 6 policies, 15 agents, 5 scenario types, but no power analysis or explicit justification for why 10 seeds or 15 agents is sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports 95% confidence intervals across seeds. §6.4.2 describes bootstrap intervals preserving dependency structure. Figure 3 shows distributions per policy."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six forgetting policies are compared: FIFO, LRU, Priority Decay, Reflection-Summary, Random Drop, and Hybrid (§6.4.1). These represent a range from naive to sophisticated baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include classical memory policies (FIFO, LRU) which are appropriate controls for this domain, plus more sophisticated approaches (Priority Decay, Reflection-Summary). Since this is a new problem formulation, there are no prior competing approaches to compare against."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The policy comparison serves as an implicit ablation: Hybrid composes temporal, reflection, importance, and privacy components, while individual policies test each component in isolation. §6.8 provides policy-specific analysis. Budget sweeps ablate capacity."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five distinct metrics are reported: Narrative Coherence, Goal Completion Rate, Social Recall Accuracy, Privacy Preservation, and Cost Efficiency (§6.2.3, Eqs. 8-13), plus a weighted Composite score."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses LLM-as-judge for narrative coherence evaluation (§6.3.1) but includes no human evaluation. §7.3 explicitly acknowledges this: 'we replace large-scale human annotation with rubricized LLM-as-judge scoring.' The paper claims human-centered evaluation but relies entirely on automated metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "§5.3 mentions 'The importance proxy Ûn is learned per type from held-out FiFA runs or specified via calibrated priors' but does not clearly describe separation of tuning and evaluation data. No explicit dev/test split is documented."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 breaks down results by policy and metric. §6.6 provides per-metric detailed analysis. §6.7 analyzes budget effects. §6.8 provides policy-specific analysis. Figure 4 shows budget×policy heatmap."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6.6.1 discusses how temporal policies remove 'bridge' episodes causing broken discourse. §6.6.2 notes extremely low goal completion rates. §6.8.3 discusses what simple policies miss. §7.2 analyzes the unexpected Random Drop result."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's most prominent finding is negative: the sophisticated Hybrid policy (0.589) loses to naive Random Drop (0.635) on the Composite. Reflection-Summary results are incomplete (footnote in Table 2). The ceiling effect in SRA is reported as a metric design failure."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'the Hybrid policy delivers the best composite performance (≈0.911)' but Table 2 shows Random Drop leads at 0.635 and Hybrid is at 0.589. No score of 0.911 appears anywhere in the results. This is a direct contradiction between abstract and results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about policy effects (e.g., 'purposeful forgetting can improve both user-facing quality'). The experimental design directly manipulates the policy variable while controlling scenario and seed, which is adequate for these claims within the simulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "§7.3 explicitly bounds generalizability: simulation vs. real deployment, English-only, specific model family, cultural scope limitations, budget range (2K-32K), retrieval-gated working set. The paper repeatedly notes these are simulation results."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§7.2 discusses how Random Drop's superiority stems from metric design (SRA ceiling, cost weighting) rather than inherent superiority. §6.5.1 discusses how reweighting the Composite changes rankings. §7.3 considers model dependence and cultural factors."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "§7.2 explicitly discusses how 'composite outcomes are sensitive to metric weights and definitions' and that 'wins are not intrinsic properties of policies but reflections of product priorities.' §6.5.1 discusses SRA ceiling effects and the gap between the metric and actual social competence."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes a system using LLMs for agent simulation and LLM-as-judge evaluation but never specifies which model or version is used. §6.3.1 mentions 'LLM-as-judge protocol' without naming the model. No model version appears anywhere."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes 'templated rubrics with auto-calibration' (§6.1.3) and 'rubricized, explanation-seeking prompts' (§6.1.4) but never provides the actual prompt text. No prompts appear in the paper or appendices."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Budget levels are specified (2K-32K tokens) and the experimental grid is described. However, critical hyperparameters are missing: α, β, γ weights for Priority Decay, λ_priv values, distortion thresholds for reflection, LLM temperature/sampling settings, and the 'calibrated priors' mentioned in §5.3."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The MaRS architecture is described in substantial detail across §§3-5 with typed memory nodes, graph structure, indices, policy execution runtime, privacy engine, and data flow (Figure 1). Policy decision flow is shown in Figure 2. The multi-layer architecture is thoroughly documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Scenario types are described at a high level (social-gathering, project-collaboration, etc. in §6.2.2), but the concrete scenario content, event generation parameters, agent population details, and the data pipeline from simulation to metric computation are not documented at a reproducible level."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§7.3 'Limitations and Constraints' provides a substantial multi-paragraph discussion covering external validity, model dependence, language scope, technical scope, formal assumptions, and methodological constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§7.3 discusses specific threats: FiFA's near-ceiling SRA may result from 'stable, redundantly encoded social facts in the simulator'; leakage opportunities are 'adversarial but fixed in frequency'; results 'are obtained with a particular family of large language models'; cultural scope limited to 'English interactions with Western conversational norms.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§7.3 explicitly states what was not tested: real human interactions, multilingual settings, budgets beyond 32K tokens, multi-modal memory, dynamic policy adaptation, adversarial attacks on memory. §7.4 frames these as future work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data from the 300 simulation runs is released. No MaRS audit logs, scenario snapshots, or per-turn metrics are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The simulation environment is described conceptually (§6.2.1-6.2.2) but lacks reproducible detail. Agent population '15-30 agents', scenario types, and stochastic scheduling are described at a high level without concrete specifications that would allow replication."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is entirely simulation-based with algorithmic agent populations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the conceptual flow from simulation to metrics but does not document the full pipeline at a reproducible level. For example, how exactly are the 5 scenario types instantiated, how are the 15 agents initialized, what are the exact event sequences, and how are metrics computed from raw logs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, grants, or acknowledgments section appears in the paper. Whether the work is funded or unfunded is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation is clearly stated: 'Computer Science Department, Al-Baha University' with email. Since the paper evaluates a novel framework rather than a commercial product, no product-affiliation conflict arises."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence. The paper provides no acknowledgments or funding statement."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates algorithmic forgetting policies (FIFO, LRU, Priority Decay, etc.) on a custom simulation benchmark. The policies are not pre-trained models whose knowledge could be contaminated by test data."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluated policies are deterministic algorithms, not pre-trained models. Train/test overlap is not a meaningful concern for FIFO, LRU, or Random Drop eviction policies."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "FiFA is a newly created benchmark and the evaluated policies are algorithmic heuristics, not pre-trained models that could have been trained on benchmark data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is entirely simulation-based."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses simulated multi-agent interactions."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Agent 'archetypes' (Social, Analytical, Creative, Practical, Empathetic) are synthetic, not human demographics."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Randomization of simulation seeds is described but this is not a human-subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports a normalized Cost Efficiency metric (§6.2.3, Eq. 13) and discusses token budgets, but never reports actual API costs, token counts consumed, wall-clock time per run, or cost per simulation step. Only relative efficiency ratios are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated: no GPU hours, API spend, hardware used, or total time for the 300 simulation runs. The paper describes O-notation complexity but reports no actual compute numbers."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 10 independent seeds per configuration. §6.4.1: 'replication strategy that executes each configuration across 10 independent seeds.' Bootstrap confidence intervals in Table 2 reflect seed variation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6.5.1 explicitly states '6 × 5 × 10 = 300 runs spanning six forgetting policies and five memory budgets.' §6.4.1 describes 10 seeds per configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. §5.3 mentions 'The proxy Ûn is learned per type from held-out FiFA runs or specified via calibrated priors' but does not describe how many configurations were tried or what search method was used."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how policy hyperparameters (α, β, γ for Priority Decay, λ_priv, distortion thresholds, reflection rate limits) were selected. §5.3 says 'the defaults match those used in the experiments' without justifying why these defaults were chosen."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "§6.4.2: 'Multiple comparisons across policies and budgets are controlled by Holm–Bonferroni correction applied per metric family.' Table 4 reports Holm–Bonferroni adjusted p-values."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed both MaRS (the architecture) and FiFA (the benchmark), and implemented all six policies. No acknowledgment of self-comparison bias. §6.3.1 validates judges but not the potential bias of authors evaluating their own framework."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Cost Efficiency is an explicit metric (§6.2.3, Eq. 13). §6.6.5 analyzes cost vs. performance trade-offs. §6.7 analyzes performance across 5 budget levels. The cost–performance frontier is a central finding (§7.2)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§7.2 discusses how composite outcomes depend on metric weights and definitions. §6.5.1 discusses ceiling effects in SRA and how reweighting changes policy rankings. The paper acknowledges FiFA's limitations as a construct for measuring agent quality and proposes opportunity-normalized variants."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares forgetting policies, not different LLM models. All policies operate within the same MaRS infrastructure. The scaffold is constant across conditions, so there is no scaffold confound to address."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The evaluated forgetting policies (FIFO, LRU, etc.) are algorithmic heuristics, not pre-trained models. Temporal leakage between training and test data is not applicable to deterministic algorithms."
       },
       "feature_leakage_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The policies being evaluated are not learned from features. They are predefined algorithms operating on memory node metadata. Feature leakage in the ML sense does not apply."
       },
       "non_independence_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The study compares algorithmic policies on a custom simulation. Train/test independence is not a meaningful concern since no model training occurs."
       },
       "leakage_detection_method": {
         "applies": false,
         "answer": false,
         "justification": "The study evaluates algorithmic memory management policies, not pre-trained models. Leakage detection methods are not applicable."
       }
     }
   },
   "red_flags": [
     {
       "flag": "Abstract contradicts results",
       "detail": "The abstract claims 'the Hybrid policy delivers the best composite performance (≈0.911)' but Table 2 shows Random Drop leads at 0.635 and Hybrid is at 0.589. No score near 0.911 appears anywhere in the results. This is a major inconsistency between the abstract and the experimental findings."
     },
     {
       "flag": "Incomplete results",
       "detail": "Footnote 1 in §6.5.1 states 'The Reflection-Summary row will be inserted once its aggregates are finalized.' One of the six policies is missing from the main results table, suggesting the paper was submitted with incomplete experiments."
     },
     {
       "flag": "No implementation released",
       "detail": "A 45-page paper describing a complex multi-layer architecture (MaRS) and benchmark (FiFA) provides no code, no data, no scenarios, and no reproduction instructions. The claims are entirely unverifiable."
     },
     {
       "flag": "Extremely low goal completion rates",
       "detail": "All policies achieve <8% goal completion (best: Random Drop 0.078). Either the benchmark is poorly calibrated, the tasks are unreasonably difficult for the system, or the metric captures something other than intended. This is not discussed as a problem."
     },
     {
       "flag": "Naive policy outperforms sophisticated approach",
       "detail": "Random Drop (random eviction) achieves the highest composite score and narrative coherence, outperforming the carefully designed Hybrid policy. While the authors discuss this, the result raises questions about whether FiFA actually measures what is claimed, or whether the metric weights were not calibrated to the paper's own motivating thesis."
     },
     {
       "flag": "No human validation of LLM-as-judge",
       "detail": "The paper claims human-centered evaluation but relies entirely on LLM-as-judge for subjective metrics like narrative coherence, with no human validation study. §7.3 acknowledges this as a limitation but proceeds to draw strong conclusions from the automated scores."
     },
     {
       "flag": "Entirely simulated evaluation",
       "detail": "All 300 experimental runs are in a controlled simulation with synthetic agents and scenarios. No real user interactions, no deployment study, no field validation. The paper's claims about trust, user experience, and human-centered AI rest entirely on simulation."
     },
     {
       "flag": "Missing model specification",
       "detail": "The paper never states which LLM is used for agent simulation or LLM-as-judge evaluation. This is a critical omission since all simulation results depend on the underlying model's behavior."
     }
   ],
   "cited_papers": [
     {
       "title": "Generative agents: Interactive simulacra of human behavior",
       "authors": ["J.S. Park", "J.C. O'Brien", "C.J. Cai", "M.R. Morris", "P. Liang", "M.S. Bernstein"],
       "year": 2023,
       "relevance": "Foundational work on LLM-based generative agents with memory and reflection, directly motivates MaRS's memory architecture design."
     },
     {
       "title": "Reflexion: language agents with verbal reinforcement learning",
       "authors": ["N. Shinn", "F. Cassano", "A. Gopinath", "K. Narasimhan", "S. Yao"],
       "year": 2023,
       "relevance": "Introduced reflection mechanisms for LLM agents that improve self-consistency, directly relevant to MaRS's reflection-summary policy."
     },
     {
       "title": "MemGPT: Towards LLMs as operating systems",
       "authors": ["C. Packer", "S. Wooders", "K. Lin", "V. Fang", "S.G. Patil", "I. Stoica", "J.E. Gonzalez"],
       "year": 2024,
       "arxiv_id": "2310.08560",
       "relevance": "OS-inspired memory paging for LLMs with working set and archival store, key comparison point for MaRS's memory management approach."
     },
     {
       "title": "MemoryBank: Enhancing large language models with long-term memory",
       "authors": ["W. Zhong", "L. Guo", "Q. Gao", "H. Ye", "Y. Wang"],
       "year": 2024,
       "doi": "10.1609/aaai.v38i17.29946",
       "relevance": "Attaches human-like decay and importance cues to LLM memory for growth stabilization."
     },
     {
       "title": "Augmenting language models with long-term memory",
       "authors": ["W. Wang", "L. Dong", "H. Cheng", "X. Liu", "X. Yan", "J. Gao", "F. Wei"],
       "year": 2023,
       "relevance": "Separates memory encoding from response generation for improved longevity and recall in LLMs."
     },
     {
       "title": "AgentBench: Evaluating LLMs as agents",
       "authors": ["X. Liu", "H. Yu", "H. Zhang", "Y. Xu"],
       "year": 2024,
       "relevance": "Major agent evaluation benchmark that FiFA positions itself against as addressing memory governance gaps."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["L. Zheng", "W.L. Chiang", "Y. Sheng"],
       "year": 2023,
       "relevance": "Establishes LLM-as-judge evaluation methodology that FiFA's rubricized scoring protocol builds upon."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2024,
       "relevance": "Cost-aware LLM evaluation framework relevant to FiFA's cost efficiency metric."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing large language model paradigm; discusses context growth and memory challenges."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Foundational LLM technical report relevant to the generative agents that MaRS aims to manage."
     },
     {
       "title": "FlashAttention-2: Faster attention with better parallelism and work partitioning",
       "authors": ["T. Dao"],
       "year": 2024,
       "relevance": "Efficient attention implementation that increases affordable context horizon, complementary to MaRS's memory governance approach."
     },
     {
       "title": "A survey on large language model based autonomous agents",
       "authors": ["L. Wang", "C. Ma", "X. Feng"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based agents that calls for principled memory budgeting and policy-level evaluation."
     },
     {
       "title": "The rise and potential of large language model based agents: A survey",
       "authors": ["Z. Xi", "W. Chen", "X. Guo"],
       "year": 2025,
       "doi": "10.1007/s11432-024-4222-0",
       "relevance": "Survey on LLM agents documenting the gap in memory governance that MaRS addresses."
     },
     {
       "title": "G-Eval: NLG evaluation using GPT-4 with better human alignment",
       "authors": ["Y. Liu", "D. Iter", "Y. Xu", "S. Wang", "R. Xu", "C. Zhu"],
       "year": 2023,
       "doi": "10.18653/v1/2023.emnlp-main.153",
       "relevance": "LLM-based evaluation methodology with human alignment, informing FiFA's rubricized judging approach."
     }
   ]
 }