ai-research-survey

calibration.json (21196B)

---

 {
   "calibration_metadata": {
     "paper_slug": "agentic-memory-learning-2026",
     "calibration_model": "opus",
     "scan_model": "sonnet",
     "calibration_date": "2026-02-28",
     "schema_version": "scan.schema.json"
   },
   "opus_checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No GitHub link or code archive is provided for AgeMem itself. The paper references third-party frameworks (AgentScope, Trinity-RFT) but the AgeMem code is not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All five benchmarks (ALFWorld, SciWorld, PDDL, BabyAI, HotpotQA) are publicly available datasets. The paper does not collect new proprietary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '8 NVIDIA RTX 4090 GPUs with 48GB memory each' (Appendix C.4) and names the frameworks (AgentScope, Trinity-RFT), but no requirements.txt, Dockerfile, or detailed dependency/version listing is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The appendix gives algorithmic pseudocode and hyperparameters, but there is no README or script-level guidance to reproduce experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 and all other result tables report only point estimates (e.g., 41.96% average). No confidence intervals, error bars, or standard deviations are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims such as 'AgeMem improves by 4.82 and 8.57 percentage points on average' are made without any statistical significance tests (no p-values, t-tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point improvements with baseline context throughout (e.g., 'relative gains of 49.59% and 23.52% over no-memory' in Section 4.2, absolute differences like '+13.9%' in ablations with both baseline and new values). Per the schema, '12% improvement over baseline (from 45% to 57%)' provides enough context for YES — the paper does this consistently."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for evaluation set sizes. HotpotQA has ~90k training questions but the exact test split size used is not stated, and no power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are single-point estimates. Despite using K=8 rollouts per task and stochastic RL training, no standard deviations, variance across seeds, or spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four baselines (LangMem, A-Mem, Mem0, Mem0g) plus a no-memory baseline and an AgeMem-noRL ablation. Links to official implementations are in Appendix C.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2025: LangMem, A-Mem (arXiv:2502.12110), Mem0 (arXiv:2504.19413). These represent current state-of-the-art memory management for LLM agents."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Figures 4 and 9 present systematic ablations isolating LTM tools, RL training, and STM tools (+LT, +LT/RL, +LT/ST/RL). Tables 4 and 5 provide reward function ablations (All-Returns vs Answer-Only)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Success Rate (SR) for ALFWorld/SciWorld/BabyAI, Progress Rate (PR) for PDDL, LLM-as-a-Judge (J) for HotpotQA, Memory Quality (MQ), and token count for context efficiency."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper using automated task completion metrics and LLM-as-a-Judge. Human evaluation of system outputs is not clearly relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states the model is 'fine-tuned with RL only on the HotpotQA training set and then evaluated directly on all datasets.' This establishes separation between training and evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-dataset breakdowns across all five benchmarks for both model backbones. Ablation figures also show per-dataset results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case studies in Appendix B compare trained vs. untrained agents in curated scenarios, but no systematic error analysis of where AgeMem fails on the benchmarks is provided. No failure mode analysis appears."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation in Figure 9b shows that adding LTM alone to Qwen3-4B on SciWorld hurts performance (-4.4%). This is reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims AgeMem 'consistently outperforms strong memory-augmented baselines across multiple LLM backbones, achieving improved task performance, higher-quality long-term memory, and more efficient context usage.' Table 2, Figure 2, and Figure 3 support these claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies isolate individual components (LTM, STM, RL) through controlled single-variable manipulation, supporting causal claims like 'RL training contributes 8.53 percentage points improvement.' These are standard ablation-based causal inferences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests on two specific Qwen model backbones (7B and 4B) on five benchmarks, but the conclusion claims results 'highlight the importance of unified, agent-centric memory policies' and 'suggest a promising direction for building scalable and adaptive LLM agents.' The title 'Agentic Memory' and broad framing do not bound generalization to the tested models and tasks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section mentions fixed tool set and limited benchmark coverage but does not discuss alternative explanations for observed improvements (e.g., whether gains come from additional RL training compute rather than the architecture, or whether benchmark selection favors the approach)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies 'Qwen2.5-7B-Instruct' and 'Qwen3-4B-Instruct' as base models, and 'Qwen-Max' as the LLM evaluator. However, per the schema, marketing names without a snapshot date or API version do NOT count. No specific model checkpoint dates, snapshot identifiers, or API versions are provided for any of these models. 'Qwen-Max' is especially vague — it is a marketing name with no version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full system prompt for AgeMem is provided verbatim in Appendix A.1, including the tool-calling format and response structure. The summarization system prompt is also fully provided. LLM-as-a-Judge and Memory Quality evaluation prompts are given in Appendix C.2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported in Appendix C.4: K=8 rollouts, KL coefficient β=0.1, uniform reward weights (1/3 each), FILTER threshold θ=0.6, max context length 8,192 tokens, max response length 2,048 tokens, penalty coefficients (Prounds=-1, Poverflow=-0.5)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The agentic scaffolding is described in detail: three-stage trajectory structure, tool interface (6 tools with exact signatures in Figures 6-7), step-wise GRPO mechanism, and complete rollout algorithms (Algorithms 1-5)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes how HotpotQA supporting facts are used as Stage 1 contextual information and mentions a DISTRACTORGEN function for generating distractors, but does not detail the distractor generation process. The exact evaluation subsets (sizes, selection criteria) for each benchmark are not documented. The data pipeline from raw benchmarks to actual training/evaluation instances is underspecified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the conclusion, discussing the fixed tool set and limited benchmark coverage."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations section is generic: it notes 'broader coverage of tasks and environments may further strengthen the empirical understanding' and the tool set 'could be extended.' No specific threats to validity are discussed (e.g., LLM judge reliability, reward hacking, same-family model bias, contamination risk)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not note that results are limited to Qwen model families, that RL training was only on HotpotQA, or that the token reduction (3-5%) may not be practically significant."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation logs, model outputs, and raw trajectory data are not released. Only aggregated performance numbers in tables and figures are available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "All five datasets are established public benchmarks with documented collection procedures from their original papers. The paper describes their structure in Appendix C.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper evaluates on standard benchmarks, so recruitment methods are not applicable."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the three-stage trajectory structure is described algorithmically, the DISTRACTORGEN function for generating distractors is mentioned but not fully specified. The exact evaluation subsets and how benchmark data is converted into the three-stage format are not fully documented. The data pipeline has unexplained steps."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "There is no acknowledgments section or funding disclosure in the paper. Authors include Alibaba Group employees but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Yi Yu, Liuyi Yao, Yuexiang Xie, Yaliang Li from Alibaba Group; others from Wuhan University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple authors are from Alibaba Group. The system uses Alibaba's AgentScope framework, is evaluated with Qwen models (Alibaba), and uses Qwen-Max as the LLM judge. The funder/employer has a direct interest in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement appears anywhere in the paper. The Alibaba affiliation is listed but no explicit declaration of financial interests or conflicts is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Qwen2.5-7B-Instruct and Qwen3-4B-Instruct but does not state their training data cutoff dates. HotpotQA (2018) and other benchmarks predate these models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark test examples appeared in Qwen model training data. Given the age of these benchmarks, contamination risk exists but is not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HotpotQA (2018), ALFWorld (2020), SciWorld (2022), BabyAI (2018) all predate the Qwen models. Contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; this is a benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Token counts per episode are reported in Figure 3 (~2,117-2,310 tokens), but no API costs, wall-clock inference time per example, or cost per episode are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '8 NVIDIA RTX 4090 GPUs with 48GB memory each' (Appendix C.4), specifying hardware. However, no training wall-clock time, total GPU-hours, or total compute budget is stated. Knowing the GPU model without knowing how long training took does not constitute a stated compute budget."
       }
     }
   },
   "comparison": {
     "total_questions": 50,
     "agreements": 45,
     "disagreements": 5,
     "agreement_rate": 0.90,
     "disagreements_detail": [
       {
         "category": "statistical_methodology",
         "question": "effect_sizes_reported",
         "sonnet_applies": true,
         "sonnet_answer": false,
         "opus_applies": true,
         "opus_answer": true,
         "direction": "opus_generous",
         "explanation": "Sonnet answered false, noting raw percentage differences but no standardized effect sizes. Opus answered true because the schema says 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' The paper consistently reports improvements with baseline context (e.g., 'relative gains of 49.59%', '+13.9%' over clearly stated baselines). This is an interpretive boundary — Sonnet was stricter by requiring formal effect size measures like Cohen's d, while Opus followed the schema's example more literally."
       },
       {
         "category": "setup_transparency",
         "question": "model_versions_specified",
         "sonnet_applies": true,
         "sonnet_answer": true,
         "opus_applies": true,
         "opus_answer": false,
         "direction": "sonnet_generous",
         "explanation": "Sonnet accepted 'Qwen2.5-7B-Instruct' and 'Qwen3-4B-Instruct' as sufficiently specified versions. However, the schema explicitly states 'Marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count as specified versions.' The Qwen model names are marketing names — there is no snapshot date, checkpoint identifier, or API version. 'Qwen-Max' (the evaluator model) is even more vague. Opus applied the schema strictly and answered false."
       },
       {
         "category": "setup_transparency",
         "question": "data_preprocessing_documented",
         "sonnet_applies": true,
         "sonnet_answer": true,
         "opus_applies": true,
         "opus_answer": false,
         "direction": "sonnet_generous",
         "explanation": "Sonnet credited the paper for describing how HotpotQA facts are used and how DISTRACTORGEN works 'conceptually.' Opus noted that DISTRACTORGEN is mentioned but not specified, the exact evaluation subsets (sizes, selection criteria) for each benchmark are not documented, and the pipeline from raw benchmarks to training/evaluation instances has unexplained steps. The schema requires that if the paper goes from 'we collected data' to 'here are the results' without describing intermediate processing, the answer is NO."
       },
       {
         "category": "data_integrity",
         "question": "data_pipeline_documented",
         "sonnet_applies": true,
         "sonnet_answer": true,
         "opus_applies": true,
         "opus_answer": false,
         "direction": "sonnet_generous",
         "explanation": "Sonnet credited the algorithmic pseudocode (Algorithms 3-5) as documenting the data pipeline. Opus noted that while the three-stage algorithm is described, the DISTRACTORGEN function is not fully specified, evaluation subset sizes are not stated, and the conversion from benchmark data to the three-stage format is not fully documented. The schema says 'if there are unexplained jumps... NO.' The distractor generation process is an unexplained step."
       },
       {
         "category": "cost_and_practicality",
         "question": "compute_budget_stated",
         "sonnet_applies": true,
         "sonnet_answer": true,
         "opus_applies": true,
         "opus_answer": false,
         "direction": "sonnet_generous",
         "explanation": "Sonnet credited '8 NVIDIA RTX 4090 GPUs with 48GB memory each' as the compute budget. However, the schema asks for 'GPU hours, total API spend, hardware used, training time.' Stating the hardware without training time or total GPU-hours does not constitute a 'total computational budget.' The reader knows the GPU model but not how long training took or the total compute consumed."
       }
     ]
   },
   "summary": {
     "agreement_rate": 0.90,
     "sonnet_generous_count": 4,
     "opus_generous_count": 1,
     "applies_boundary_count": 0,
     "interpretive_count": 0,
     "dominant_error_type": "sonnet_generous",
     "notes": "Of 5 disagreements, 4 are sonnet_generous (Sonnet credited the paper where Opus found it insufficient under strict schema reading) and 1 is opus_generous (Opus credited effect sizes based on the schema's example of contextual percentage improvements). The pattern continues the known Sonnet generosity bias: Sonnet tends to credit partial or conceptual descriptions (data_preprocessing_documented, data_pipeline_documented) and incomplete specifications (model_versions_specified, compute_budget_stated) as satisfying the criteria. Key issues: (1) model_versions_specified — Sonnet accepted marketing-style model names without snapshot dates, contradicting the schema's explicit prohibition; (2) compute_budget_stated — hardware alone without GPU-hours or training time is not a budget; (3) data pipeline items — mentioning a function name (DISTRACTORGEN) without specifying it does not count as documentation."
   }
 }