ai-research-survey

scan.json (21722B)

---

 {
   "paper": {
     "title": "Grokking: Generalization Beyond Overfitting on Small Algorithmic Datasets",
     "authors": ["Alethea Power", "Yuri Burda", "Harri Edwards", "Igor Babuschkin", "Vedant Misra"],
     "year": 2022,
     "venue": "arXiv.org",
     "arxiv_id": "2201.02177"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Neural networks trained on small algorithmic datasets (binary operation tables) exhibit 'grokking' — generalization that occurs well after memorization/overfitting, sometimes requiring 1000x more optimization steps. The amount of optimization required for generalization grows rapidly as dataset size decreases. Weight decay is the most effective intervention for improving data efficiency, more than halving the required training data. Embedding visualizations reveal that generalizing networks learn meaningful mathematical structure (e.g., circular topology for modular arithmetic).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code release is mentioned anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The datasets are algorithmically generated from fully specified binary operations (listed in Appendix A.1.1) with explicit parameters (p=97, S5). Anyone can reproduce the exact data from the specification."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, or dependency information is provided. The paper mentions using a transformer but does not specify the framework or environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Hyperparameters are listed in Appendix A.1.2 but there are no scripts or README-level instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as medians or means across seeds but no confidence intervals or error bars are shown in the figures or text."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported. Claims about weight decay being more effective are based on visual comparison of data efficiency curves."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes: 'a decrease of 1% of training data leads to an increase of 40-50% in median time to generalization' (Section 3.1.1), and weight decay 'more than halving the amount of samples needed' (Section 3.3)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Experiments use 3 random seeds (7 for Section 3.1.1) but no justification is given for why these numbers are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 3 seeds (Figure 2 right) or medians over 7 seeds (Section 3.1.1), but no standard deviations, IQR, or spread measures are reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 and Figure 2 (left) compare multiple optimization algorithms (Adam, AdamW, with/without dropout, weight noise, gradient noise, different learning rates) as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are standard optimization methods (Adam, AdamW with weight decay) that were contemporary at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 provides ablations over optimization methods, learning rates, weight decay, dropout, and gradient noise, measuring their individual impact on data efficiency."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy and loss (training and validation) as evaluation metrics, shown in Figures 1 and 4."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant — the paper studies neural network generalization on algorithmic datasets with ground-truth answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For each run, a fraction of equations is the training set and the rest is the validation set (Appendix A.1.1). Results are reported on the held-out validation portion."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 (right) provides per-operation breakdowns across 12 different binary operations, showing data efficiency curves for each."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that some operations (e.g., x³ + xy² + y mod 97) 'didn't lead to generalization within the allowed optimization budget at any percentage of data up to 95%' (Section 3.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that some operations failed to generalize entirely, and that suboptimal hyperparameters 'severely limit generalization' (Figure 2 left caption). Section A.4 shows outliers hindering generalization."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims about grokking, data efficiency, and optimization time are all supported by the experimental results in Sections 3.1-3.3 and corresponding figures."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'weight decay improves generalization') are supported by controlled ablation experiments where only the intervention variable is changed (Section 3.3, Figure 2 left)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are appropriately bounded to 'small algorithmic datasets' and 'binary operation tables.' The paper explicitly frames these as testbeds, not general claims about all neural network training."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 and Appendix A.3 discuss whether grokking is the same as double descent (concluding it may be distinct), and Section A.5 explores sharpness as a potential explanatory mechanism. The paper considers the role of noise in SGD driving optimization to flatter minima."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exactly what it claims: validation accuracy and loss on held-out equations from binary operation tables. There is no proxy gap — 'generalization' is directly measured as performance on unseen equations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The model is fully specified: 'a standard decoder-only transformer with 2 layers, width 128, and 4 attention heads, with a total of about 4·10⁵ non-embedding parameters' (Appendix A.1.2). This is a custom model, not an API model requiring version numbers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting — it trains a transformer from scratch on equation tokens."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1.2 provides full hyperparameters: AdamW with lr=10⁻³, weight decay=1, β₁=0.9, β₂=0.98, warmup over first 10 updates, minibatch size 512, and optimization budget."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper trains a standard transformer."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1.1 fully documents the data generation: binary operations are specified, equations are tokenized as ⟨x⟩⟨op⟩⟨y⟩⟨=⟩⟨x◦y⟩, and random fractions are selected for train/validation splits."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion (Section 4) mentions future work but does not systematically discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address whether grokking is architecture-dependent or whether the hyperparameter choices bias the results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly frames its scope as 'small algorithmically generated datasets' and 'binary operation tables,' and states these are 'testbeds for theories of generalization' rather than claiming broad applicability."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (training curves, model checkpoints) is released. The datasets are algorithmically generated but the experimental outputs are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1.1 fully describes data generation: all 12 binary operations are specified with parameters, and the equation format is documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is algorithmically generated from standard mathematical operations."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from operation definition → equation generation → tokenization → train/validation split is fully documented in Section 2 and Appendix A.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present. The authors are from OpenAI and Google."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: OpenAI for four authors, Google for Vedant Misra (with note that he was at OpenAI during the work)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work was conducted at OpenAI, which has a commercial interest in demonstrating neural network capabilities. No independent funding source is mentioned."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains models from scratch on algorithmically generated data. There is no pre-trained model whose training cutoff could cause contamination."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated on a benchmark. The train/test split is a random partition of the generated equations — overlap is impossible by construction."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model or public benchmark is used. Models are trained from scratch on synthetic data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or training cost is reported, despite experiments running up to 10⁶ optimization steps."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions experiments 'can be quickly reproduced on a single GPU' but does not quantify GPU hours or total compute."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results in Figure 2 (right) are 'averaged over 3 seeds' and Section 3.1.1 uses 7 random seeds with median reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1.2: 'We've repeated each experiment for each dataset size with 3 random seeds, with the exception of experiments in section 3.1.1, where we've aggregated results over 7 random seeds.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.1.2 mentions 'We have tuned optimization hyperparameters by running experiments on modular addition and product in S5' but does not report how many configurations were tried or the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1.2 states the final configuration was 'a balance of performance we saw on S5 and simplicity,' providing justification for the selection criterion."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of self-comparison bias. All baselines are the authors' own implementations of standard optimizers."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 (center) and Section 3.1.1 explicitly show performance (time to generalization) as a function of dataset size/compute, and Section 3.3 compares methods at matched optimization budgets (10⁵ steps)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses what the algorithmic datasets test (generalization on binary operation tables) and argues they are useful testbeds for studying generalization phenomena, with clear acknowledgment of the scope."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — the paper trains a standard transformer from scratch."
       }
     }
   },
   "claims": [
     {
       "claim": "Neural networks exhibit 'grokking': generalization that occurs well after overfitting, sometimes requiring 1000x more optimization steps than needed for memorization.",
       "evidence": "Figure 1 (left) shows training accuracy reaching near-perfect at <10³ steps while validation accuracy requires ~10⁶ steps for modular division mod 97.",
       "supported": "strong"
     },
     {
       "claim": "The amount of optimization required for generalization increases rapidly as dataset size decreases.",
       "evidence": "Figure 1 (center) shows median optimization steps to 99% validation accuracy on S5, with a 1% decrease in training data leading to 40-50% increase in optimization time (Section 3.1.1).",
       "supported": "strong"
     },
     {
       "claim": "Weight decay is the most effective regularization for improving data efficiency on these tasks, more than halving the required samples.",
       "evidence": "Figure 2 (left) compares multiple optimization methods on S5, showing weight decay substantially shifts the data efficiency curve left compared to all other interventions.",
       "supported": "moderate"
     },
     {
       "claim": "Symmetric operations require less data for generalization than non-symmetric counterparts.",
       "evidence": "Figure 2 (right) shows x+y and x*y requiring lower data fractions than x-y and x/y respectively, consistent across multiple operation pairs.",
       "supported": "moderate"
     },
     {
       "claim": "Sharpness of the loss landscape is predictive of grokking, with Spearman correlation of -0.79548 (p < 0.000014).",
       "evidence": "Appendix A.5 reports this correlation between validation accuracy and sharpness measure φ on S5 composition, with Figure 7 visualizing the relationship.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or spread measures reported",
       "detail": "Despite running 3-7 seeds per experiment, the paper reports only means or medians without standard deviations, IQR, or error bars, making it impossible to assess result stability."
     },
     {
       "flag": "Narrow architecture evaluation",
       "detail": "All experiments use a single small transformer architecture (2 layers, 128 width, 4 heads). Claims about grokking being a general phenomenon rest on a single architecture, though the paper acknowledges some effects 'might be partially architecture-dependent.'"
     }
   ],
   "cited_papers": [
     {
       "title": "Deep Double Descent: Where Bigger Models and More Data Hurt",
       "authors": ["Preetum Nakkiran", "Gal Kaplun", "Yamini Bansal", "Tristan Yang", "Boaz Barak", "Ilya Sutskever"],
       "year": 2019,
       "arxiv_id": "1912.02292",
       "relevance": "Foundational work on double descent in neural networks, directly related to the grokking phenomenon studied here."
     },
     {
       "title": "Understanding deep learning requires rethinking generalization",
       "authors": ["Chiyuan Zhang", "Samy Bengio", "Moritz Hardt", "Benjamin Recht", "Oriol Vinyals"],
       "year": 2016,
       "arxiv_id": "1611.03530",
       "relevance": "Key paper on generalization in overparameterized networks — shows networks can memorize random labels, foundational to understanding grokking."
     },
     {
       "title": "Attention is all you need",
       "authors": ["Ashish Vaswani", "Noam Shazeer", "Niki Parmar", "Jakob Uszkoreit", "Llion Jones", "Aidan N Gomez", "Lukasz Kaiser", "Illia Polosukhin"],
       "year": 2017,
       "arxiv_id": "1706.03762",
       "relevance": "Introduced the transformer architecture used in all experiments in this paper."
     },
     {
       "title": "Fantastic generalization measures and where to find them",
       "authors": ["Yiding Jiang", "Behnam Neyshabur", "Hossein Mobahi", "Dilip Krishnan", "Samy Bengio"],
       "year": 2019,
       "arxiv_id": "1912.02178",
       "relevance": "Studies which generalization measures predict neural network generalization — directly relevant to understanding why grokking occurs."
     },
     {
       "title": "Analysing mathematical reasoning abilities of neural models",
       "authors": ["David Saxton", "Edward Grefenstette", "Felix Hill", "Pushmeet Kohli"],
       "year": 2019,
       "arxiv_id": "1904.01557",
       "relevance": "Studies neural network generalization on procedurally generated math problems, closely related to the algorithmic datasets used here."
     }
   ]
 }