ai-research-survey

scan.json (18458B)

---

 {
   "paper": {
     "title": "Grokking modular arithmetic",
     "authors": ["Andrey Gromov"],
     "year": 2023,
     "venue": "arXiv preprint",
     "arxiv_id": "2301.02679",
     "doi": "10.48550/arXiv.2301.02679"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "The paper presents analytic weight solutions for two-layer fully-connected networks that solve modular addition and related tasks, exhibiting grokking under vanilla gradient descent with MSE loss and no regularization. The learned features are periodic functions with frequencies determined by the modular base, and grokking corresponds to the network learning these specific Fourier features. Width increases accuracy through better destructive interference, and regularization/adaptive optimizers reduce grokking time and data requirements but are not necessary.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository URL or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The data is deterministically generated modular arithmetic tables (e.g., all pairs (n,m) in Z_p). The generation procedure is fully specified — no external dataset is needed."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, library versions, or dependency information is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or scripts are provided. The setup is described mathematically but there are no concrete implementation details."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are presented as single curves and point estimates (e.g., accuracy vs. width, learning curves) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative claims between competing methods — it presents analytic solutions and shows gradient descent finds them. No statistical tests are needed."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper is primarily theoretical, presenting analytic solutions rather than comparative effect sizes."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper. The dataset size is the full modular arithmetic table (p^2 points), which is deterministic, not sampled."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Experimental results (learning curves, accuracy vs. width) appear to be single-run results with no variance or standard deviation reported across runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares GD, AdamW, and the analytic solution, and discusses prior work using transformers and other architectures on the same tasks."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines reference contemporary work from 2022: Power et al. (original grokking), Liu et al., Thilak et al., all from 2022."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper varies multiple components: activation functions (quadratic, ReLU, GELU), optimizers (GD, AdamW, SGD with various regularizers), width N, and data fraction α (Fig. 4, Fig. 6)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports train/test loss, train/test accuracy, weight norms, gradient norms, IPR (inverse participation ratio), and Fourier analysis of preactivations."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to a theoretical paper about neural network dynamics on synthetic data."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is explicitly split into train (D_train) and test (D_test) subsets with fraction α = |D_train|/|D| (Eq. 3), and test accuracy/loss are reported separately."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by modular function type (n+m, n²+m², (n+m)², nm, n²+m²+nm, n³+nm²+m) with different behaviors noted for each (Section 3.2, Appendix C)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses functions that cannot be grokked (f(n,m) = n³+nm²+m never rises above 1% test accuracy, Fig. 10) and functions with incomplete grokking (n²+m²+nm reaches ~97% with a persistent train-test gap)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that certain functions cannot be learned (n³+nm²+m), that the analytic solution for modular square root only achieves ~50% accuracy due to non-invertibility, and that no analytic solution was found for multiplication."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims (i) grokking with MSE/no regularization, (ii) feature maps determined by task, (iii) analytic weight expressions, (iv) GD/AdamW find these features — are all supported by the analytic derivations and experiments in Sections 2-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about what features the network learns and why grokking occurs. These are justified by the analytic solution (constructive/destructive interference mechanism) and empirical verification that GD finds the same solution (Fig. 2, Fig. 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly bounds results to specific function classes (f1(n)+f2(m) mod p and F(f1(n)+f2(m)) mod p), explicitly states which functions it cannot solve analytically (multiplication), and which functions it cannot grok at all."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 reviews competing explanations for grokking (slingshot mechanism, encoder-decoder competition, regularization necessity) and Section 5.1 addresses them: e.g., grokking occurs without learnable encoders contrary to Liu et al., and without regularization contrary to Nanda & Lieberum."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures test accuracy on modular arithmetic tasks and claims the network learns modular arithmetic — the measurement matches the claim with no proxy gap."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own simple neural networks from scratch — no pre-trained models or API versions are involved."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It trains neural networks directly."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Key hyperparameters like learning rate, number of epochs, and specific optimizer settings are not explicitly reported. Only α (data fraction) and N (width) are specified for experiments."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Data preprocessing is fully described: one-hot encoding of integers in Z_p, input dimension 2p, output dimension p, random train/test split with fraction α (Section 2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Discussions' extensively discusses open problems including complexity classes of modular functions, scaling to many variables, role of depth, and connection to real-world datasets."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies specific limitations: analytic solutions only work for single-hidden-layer networks, the exact solution for multiplication is unknown, certain functions cannot be grokked at all, and the connection to real-world feature learning is speculative."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states which function classes are solved (f1(n)+f2(m) mod p) vs. not (multiplication, mixed additive-multiplicative), that solutions are for single-hidden-layer networks only, and that the analytic solution is approximate (improving with width)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": false,
         "answer": false,
         "justification": "Data is deterministically generated modular arithmetic tables — no raw data release is needed as anyone can reconstruct it from the description."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data generation is fully specified: all p² pairs (n,m) for modular arithmetic over Z_p, with one-hot encoding (Section 2)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data is synthetic/deterministic."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is simple and fully documented: generate all p² data points, one-hot encode, split into train/test with fraction α."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses NSF CAREER Award DMR-2045181, Sloan Foundation, and Laboratory for Physical Sciences through the Condensed Matter Theory Center."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliation with Meta AI and University of Maryland is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are NSF, Sloan Foundation, and Laboratory for Physical Sciences — none have a financial stake in the grokking results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. The author works at Meta AI which develops neural networks, but no financial interest disclosure is made."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains its own networks from scratch on synthetic data — no pre-trained model evaluation on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation. Train/test split is controlled by the authors."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper. Cost is irrelevant."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical paper studying toy neural networks."
       }
     }
   },
   "claims": [
     {
       "claim": "Fully-connected two-layer networks exhibit grokking on modular arithmetic tasks under vanilla gradient descent with MSE loss and no regularization.",
       "evidence": "Fig. 1 shows training dynamics with delayed generalization onset. Section 2 describes the minimal setup. Multiple modular functions are tested.",
       "supported": "strong"
     },
     {
       "claim": "Analytic weight expressions (periodic cosine functions) solve modular addition and the class f(n,m) = f1(n) + f2(m) mod p.",
       "evidence": "Claim I (Eqs. 6-7) with full derivation through Eqs. 8-16 showing constructive/destructive interference mechanism. Claim II (Eq. 18) generalizes to arbitrary single-variable functions.",
       "supported": "strong"
     },
     {
       "claim": "Gradient descent and AdamW find approximately the same solution as the analytic one.",
       "evidence": "Fig. 2 shows matching Fourier structure between GD-found and analytic preactivations. Fig. 3 shows the phase constraint ϕ1+ϕ2−ϕ3≈0 is satisfied by trained networks.",
       "supported": "strong"
     },
     {
       "claim": "Regularization and adaptive optimizers reduce grokking time and critical data fraction but are not necessary for grokking.",
       "evidence": "Fig. 4a shows grokking time vs. data fraction for GD, GD+momentum, AdamW. Section 2 states grokking occurs with full-batch GD and no regularization.",
       "supported": "strong"
     },
     {
       "claim": "Some modular functions (e.g., n³+nm²+m) cannot be grokked by the tested architectures.",
       "evidence": "Fig. 10 shows generalization never rises above 1% for this function. Section 3.2 discusses complexity classes of modular functions.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or multiple-run results",
       "detail": "All experimental results appear to be single runs. For a paper about training dynamics which can be sensitive to initialization and hyperparameters, the absence of variance across seeds is a concern."
     },
     {
       "flag": "Missing hyperparameter details",
       "detail": "Key training hyperparameters (learning rate, number of epochs, specific optimizer parameters beyond what's shown in figure captions) are not systematically reported, making reproduction difficult."
     }
   ],
   "cited_papers": [
     {
       "title": "Grokking: Generalization beyond overfitting on small algorithmic datasets",
       "authors": ["Alethea Power", "Yuri Burda", "Harri Edwards", "Igor Babuschkin", "Vedant Misra"],
       "year": 2022,
       "arxiv_id": "2201.02177",
       "relevance": "Original grokking discovery paper studying how transformers learn algorithmic datasets — foundational work on delayed generalization."
     },
     {
       "title": "Towards understanding grokking: An effective theory of representation learning",
       "authors": ["Ziming Liu", "Ouail Kitouni", "Niklas Nolte", "Eric J Michaud", "Max Tegmark", "Mike Williams"],
       "year": 2022,
       "arxiv_id": "2205.10343",
       "relevance": "Develops theoretical framework for grokking as encoder-decoder competition — relevant to understanding neural network representation learning."
     },
     {
       "title": "A mechanistic interpretability analysis of grokking",
       "authors": ["Neel Nanda", "Tom Lieberum"],
       "year": 2022,
       "relevance": "Reverse-engineers the algorithm learned by transformers for modular addition — directly relevant to mechanistic interpretability of neural networks."
     },
     {
       "title": "Hidden progress in deep learning: SGD learns parities near the computational limit",
       "authors": ["Boaz Barak", "Benjamin L Edelman", "Surbhi Goel", "Sham Kakade", "Eran Malach", "Cyril Zhang"],
       "year": 2022,
       "arxiv_id": "2207.08799",
       "relevance": "Theoretical analysis of grokking on sparse parity problems with scaling laws for grokking time."
     },
     {
       "title": "Neural tangent kernel: Convergence and generalization in neural networks",
       "authors": ["Arthur Jacot", "Franck Gabriel", "Clément Hongler"],
       "year": 2018,
       "relevance": "Foundational work on NTK theory — relevant because grokking requires feature learning beyond the NTK regime."
     }
   ]
 }