ai-research-survey

scan.json (25954B)

---

 {
   "paper": {
     "title": "Nested Learning: The Illusion of Deep Learning Architectures",
     "authors": ["Ali Behrouz", "Meisam Razaviyayn", "Peilin Zhong", "Vahab Mirrokni"],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2512.24695",
     "doi": "10.48550/arXiv.2512.24695"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: CLINC, Banking, DBpedia, RULER, BABILong, MTOB, Manchu, FineWeb-Edu, ImageNet-21K, and standard common-sense reasoning datasets. No proprietary datasets were created."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. Hardware and library versions are not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1-6 and Figures 6-12 report point estimates only, with no confidence intervals or error bars on the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Hope outperforms baselines across multiple benchmarks but provides no statistical significance tests. Comparisons are made by comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context, showing absolute performance numbers for both the proposed method and baselines (e.g., Table 2 shows perplexity and accuracy for all models), allowing readers to compute relative improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of tokens used for training (30B/100B), the number of benchmark instances evaluated, or why these sample sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or spread measures are reported for any experiment. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included across all experiments: Transformer++, Samba, RetNet, DeltaNet, RWKV-7, Comba, TTT, Miras, DLA, Titans, GPT-4, Llama-8B, RMT, ARMT, and others (Tables 1-5, Figures 6-9)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include very recent models: RWKV-7 (2025), Comba (2025), Titans (2025), Miras (2025), DLA (2025), GPT-4o-mini, all published in 2024-2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 provides an ablation study removing individual components: DGD, momentum, weight decay, CMS, and inner projections for k, v, q, showing each contributes to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: perplexity (Wikitext, LMB), accuracy (PIQA, HellaSwag, WinoGrande, ARC, SIQA, BoolQ), ChRF for translation, and task-specific metrics for NIAH, BABILong, MAD, and formal language tasks."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to the claims made here, which are about architecture design, memory systems, and performance on automated benchmarks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test sets are used (RULER, BABILong, common-sense reasoning benchmarks). The paper follows established evaluation protocols from prior work."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per task across multiple tables: NIAH variants (S-NIAH-1/2/3, MK/MQ/MV) in Table 1, individual reasoning benchmarks in Table 2, individual recall tasks in Tables 3-4, and formal language tasks in Table 5."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations: Hope's MK-NIAH and multi-value performance lags behind Transformers (Table 1), short in-context recall tasks show a gap with Transformers (Table 3), and M3 optimizer has computational overhead (Figure 12, Section 10)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study (Table 6) shows components that hurt performance when removed. The paper also notes that M3 optimizer is slower than Muon (Figure 12) and that Hope underperforms Transformers on certain recall tasks (Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'promising results in language modeling, knowledge incorporation, and few-shot generalization tasks, continual learning, and long-context reasoning tasks,' which are supported by experimental sections 9.1-9.6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'CMS design helps with continual learning' are supported by controlled ablation experiments (Table 6, Figure 7) with single-variable manipulation. The paper isolates individual components and measures their effect."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The Illusion of Deep Learning Architectures' and broad claims about 'all modern architectures' being uniform are not bounded to the tested settings. Experiments use models up to 1.3B parameters, but claims extend to general deep learning."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for why Hope outperforms baselines. Could the improvements come from simply having more parameters? Different training dynamics? These are not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures specific benchmarks and generally states results in terms of those benchmarks (e.g., 'performance on BABILong', 'accuracy on common-sense reasoning'). Claims about 'continual learning' are tested with specific continual learning benchmarks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model sizes are specified (760M, 1.3B parameters) and backbone models are named (Llama3-8B, Llama-3B for continual learning experiments). GPT-4 and GPT-4o-mini are used as baselines in BABILong."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting — it trains and evaluates neural architectures directly on their respective tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training details include: vocabulary size 32K, AdamW optimizer, 'tuned learning rate for each model,' 'default optimizer configuration in Behrouz et al. (2025c),' 50B tokens for NIAH experiments, 30B/100B tokens for LM experiments, 15B tokens for continual pre-training."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper presents neural architecture designs trained end-to-end."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Training data is described as 'a mixture of FineWeb-Edu and long-context documents' but the mixing ratio, any filtering criteria, and document selection process are not described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Conclusion) includes substantive discussion: 'Is Catastrophic Forgetting Solved?' explicitly states 'the undesirable phenomenon of catastrophic forgetting is not solved in general' and discusses compression-forgetting tradeoff."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The limitations discussion is about conceptual scope ('NL as a roadmap rather than a destination') rather than specific threats to validity of the experiments. No mention of specific experimental threats like training instability, hyperparameter sensitivity, or benchmark selection bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states specific scope boundaries: 'we focus on the first stage: memory consolidation as an online process' (Section 1.1), notes M3 'might suffer from computational overhead and so face challenges when scaling to larger networks' (Section 7.2), and excludes comparison with Cartridges 'due to fundamental differences in computational costs' (Section 9.1)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, training logs, per-example predictions) is made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Benchmark datasets are referenced with citations: CLINC, Banking, DBpedia with descriptions (Section 9.1), RULER and BABILong with protocol references, FineWeb-Edu and ImageNet-21K with citations."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw data (FineWeb-Edu + long-context documents) to training is not documented. No filtering criteria, mixing ratios, or preprocessing steps are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors have Google email addresses but no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed via email addresses: three authors at Google ({alibehrouz, razaviyayn, mirrokni}@google.com) and one at Columbia University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of four authors are at Google, which has direct commercial interest in novel architecture designs. Google is not independent of the outcome. No explicit funding disclosure to evaluate further."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Models are trained from scratch on FineWeb-Edu data, but the temporal coverage of this dataset is not stated. For continual learning experiments using Llama3-8B/3B, no training cutoff is mentioned."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether FineWeb-Edu training data could overlap with benchmark test sets (e.g., CLINC, Banking, DBpedia, common-sense reasoning datasets)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Many benchmarks used (PIQA, HellaSwag, ARC, BoolQ, etc.) were published years before the training data was collected. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 12 compares training time/efficiency of M3 vs Muon vs AdaMuon at 140M and 1.3B scales. Section 7.1 analyzes computational overhead of CMS. Section 8.2 discusses parallelizable training."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training budgets stated: 30B tokens for 760M models, 100B tokens for 1.3B models, 50B tokens for NIAH experiments, 15B tokens for continual pre-training. Figure 12 shows wall-clock training times."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multi-seed experiments reported. All results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never explicitly stated for any experiment."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'tuned learning rate for each model' but does not disclose the search budget, method, or number of configurations tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the best configuration was selected. Learning rates were 'tuned' but selection procedure not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied despite comparisons across many models and benchmarks."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors propose Hope and compare against their own implementations of baselines (all trained from scratch with the same data). The bias of authors implementing and tuning their own system vs baselines is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 12 directly compares training time of different optimizers. Section 7.1 discusses the compute overhead of CMS. Section 9.1 excludes Cartridges comparison due to 'fundamental differences in their computational costs.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure the claimed capabilities (e.g., whether NIAH tests 'continual learning' vs just 'long context retrieval')."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated directly on benchmarks."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. Many benchmarks (PIQA 2020, HellaSwag 2019, ARC 2018) predate the training data but this is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setup leaks information. For fine-tuned models on BABILong, no analysis of whether fine-tuning data overlaps with test."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data across any experiments."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used or described."
       }
     }
   },
   "claims": [
     {
       "claim": "Hope outperforms all baselines on average in language modeling and common-sense reasoning at both 760M and 1.3B parameter scales.",
       "evidence": "Table 2 shows Hope achieves 52.28 avg accuracy at 760M/30B and 58.04 at 1.3B/100B, compared to next best Titans at 51.68 and 56.82 respectively. Perplexity also lowest.",
       "supported": "moderate"
     },
     {
       "claim": "Hope with CMS design maintains performance at 10M context length on BABILong while other models degrade.",
       "evidence": "Figure 9 shows Hope maintaining performance at 10M context while Titans and ARMT drop fast after 1M. However, this requires fine-tuning.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-level memory (CMS) improves continual learning over single-level in-context learning.",
       "evidence": "Figure 6 shows Hope outperforms ICL, EWC, and InCA on class-incremental learning across CLINC, Banking, and DBpedia. Figure 7 shows increasing levels improves NIAH, LongHealth, and QASPER.",
       "supported": "moderate"
     },
     {
       "claim": "Known gradient-based optimizers (Adam, SGD with Momentum) can be reformulated as associative memory modules.",
       "evidence": "Sections 4.1-4.2 and Appendix B provide mathematical derivations showing Adam, momentum SGD, AdaGrad, and related optimizers as associative memories compressing gradients. The derivations are rigorous.",
       "supported": "strong"
     },
     {
       "claim": "M3 optimizer finds more effective solutions than AdamW and Muon on ImageNet ViT training.",
       "evidence": "Figure 11 shows M3 achieves lower training and test loss on ViT 24M and 86M models on ImageNet-21K. However, single-run results with no error bars.",
       "supported": "weak"
     },
     {
       "claim": "Hope achieves perfect scores on all formal language recognition tasks while Transformers fail.",
       "evidence": "Table 5 shows Hope gets 100% on all tasks (Parity, Non-Star-Free Regular, Counter, Parallel, Shuffle-2) while Transformer fails on Parity and Non-Star-Free Regular. However, LSTM and SRWM also achieve perfect scores.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper presents Nested Learning, a paradigm that reframes deep learning models as interconnected systems of nested optimization problems with different update frequencies. It shows that popular optimizers (Adam, SGD with momentum) and architectures (Transformers, linear attention) can be unified as associative memories compressing their own context flow. The proposed Hope architecture, combining self-modifying Titans with Continuum Memory Systems, demonstrates improvements in continual learning, long-context reasoning (maintaining performance at 10M context on BABILong), and language modeling over existing architectures including Transformers and modern RNNs.",
   "red_flags": [
     {
       "flag": "No error bars or multi-seed experiments",
       "detail": "All experimental results across Tables 1-6 and Figures 6-12 appear to be single-run point estimates. For architecture comparisons where hyperparameters were tuned per model, seed sensitivity could significantly affect conclusions."
     },
     {
       "flag": "Authors evaluate their own system with self-implemented baselines",
       "detail": "All models are trained from scratch by the authors. The bias of authors implementing and tuning their own system vs baselines is well-documented (Lucic et al. 2018) but not acknowledged."
     },
     {
       "flag": "Broad theoretical claims beyond experimental scope",
       "detail": "The title claims 'The Illusion of Deep Learning Architectures' and the paper makes sweeping claims about all modern architectures being uniform. Experiments are limited to models up to 1.3B parameters and a specific set of benchmarks."
     },
     {
       "flag": "Company affiliation evaluating novel architectures",
       "detail": "Three of four authors are Google employees. Google has commercial interest in novel architectures. No competing interests statement is provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Attention is All you Need",
       "authors": ["Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit", "Llion Jones", "Aidan N Gomez", "Łukasz Kaiser", "Illia Polosukhin"],
       "year": 2017,
       "relevance": "Foundational Transformer architecture paper; NL reinterprets attention as a non-parametric associative memory."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "relevance": "Defines in-context learning in LLMs; NL reframes ICL as a consequence of multi-level nested optimization."
     },
     {
       "title": "Titans: Learning to Memorize at Test Time",
       "authors": ["Ali Behrouz", "Peilin Zhong", "Vahab Mirrokni"],
       "year": 2025,
       "relevance": "Direct predecessor to Hope architecture; introduces deep memory modules that Hope extends with self-modification and CMS."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "relevance": "Establishes neural scaling laws that NL reinterprets through the lens of nested optimization and parameter capacity across levels."
     },
     {
       "title": "Adam: A method for stochastic optimization",
       "authors": ["Diederik P Kingma", "Jimmy Ba"],
       "year": 2014,
       "relevance": "NL shows Adam is an optimal associative memory for L2 regression on gradients, reframing the most widely used optimizer."
     },
     {
       "title": "Learning to (learn at test time): RNNs with expressive hidden states",
       "authors": ["Yu Sun"],
       "year": 2024,
       "relevance": "TTT framework for test-time training; NL reframes this as parametric in-context learning within the nested paradigm."
     },
     {
       "title": "It's All Connected: A Journey Through Test-Time Memorization, Attentional Bias, Retention, and Online Optimization",
       "authors": ["Ali Behrouz", "Meisam Razaviyayn", "Peilin Zhong", "Vahab Mirrokni"],
       "year": 2025,
       "arxiv_id": "2504.13173",
       "relevance": "Miras framework unifying sequence models as associative memories; direct theoretical foundation for NL."
     },
     {
       "title": "RWKV-7 'Goose' with Expressive Dynamic State Evolution",
       "authors": ["Bo Peng"],
       "year": 2025,
       "arxiv_id": "2503.14456",
       "relevance": "Modern RNN architecture used as baseline; NL positions it within the nested associative memory framework."
     },
     {
       "title": "Muon: An optimizer for hidden layers in neural networks",
       "authors": ["K Jordan", "Y Jin", "V Boza"],
       "year": 2024,
       "relevance": "Optimizer that NL reinterprets as gradient orthogonalization via nested optimization; basis for M3 optimizer."
     },
     {
       "title": "Model-agnostic meta-learning for fast adaptation of deep networks",
       "authors": ["Chelsea Finn", "Pieter Abbeel", "Sergey Levine"],
       "year": 2017,
       "relevance": "MAML as knowledge transfer via initialization; NL subsumes meta-learning as a special case of nested optimization."
     }
   ]
 }