ai-research-survey

scan.json (26888B)

---

 {
   "paper": {
     "title": "Disaggregation Reveals Hidden Training Dynamics: The Case of Agreement Attraction",
     "authors": [
       "James A. Michaelov",
       "Catherine Arnett"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2510.24934",
     "doi": "10.48550/arXiv.2510.24934"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Disaggregating language model performance on subject-verb agreement benchmarks by condition reveals hidden 'breakthroughs' in training that are invisible in aggregate metrics. Models first learn frequency-based heuristics (preferring more common verb forms), then become sensitive to local context (bigram-like behavior producing agreement attraction effects), and finally improve overall. These phases proceed through rapid non-monotonic transitions rather than the gradual improvement suggested by aggregate scores.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 (Procedure) states: 'We release all code in the following repository: https://github.com/jmichaelov/sv-disaggregation-cognitive-interpretability.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: BIG-bench Subject-Verb Agreement task subsets and stimuli from Bock and Cutting (1992) as preprocessed by Arehalli and Linzen (2020). The PolyPythia models are also publicly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements.txt, or dependency versions are mentioned in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the paper does not include step-by-step reproduction instructions. The procedure section describes the method conceptually but not as a reproducible recipe."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1-6 include shading reflecting '95% confidence intervals' as stated in the figure captions."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims about performance differences across conditions and training phases but does not report any statistical significance tests. Differences are described qualitatively from visual inspection of plots."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. Performance differences are described qualitatively (e.g., 'sharp increase', 'drop in accuracy') without quantifying magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 10 random seeds per model size is sufficient, nor any power analysis. The choice of 5 model sizes is not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The PolyPythia suite provides 10 random seeds per model size, and the 95% confidence interval shading in all figures reflects variance across seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The aggregate (mean across conditions) score serves as the baseline comparison point. The paper's core contribution is showing how disaggregated condition-level results differ from this aggregate."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "This is not a system comparison paper proposing a new method against prior methods. It analyzes training dynamics of existing models on existing benchmarks."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate. The paper is an analysis study, not a system proposal."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only accuracy (whether the model assigns higher log-probability to the correct verb form) is used. The paper explicitly notes in the conclusion it uses 'a simplistic evaluation metric such as accuracy.'"
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to this paper's claims about model training dynamics."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No model training or tuning is performed. The study evaluates pre-existing model checkpoints on fixed datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "This is the core contribution. Results are broken down by verb type (be vs. single-token vs. multi-token), by individual verb (Appendix B), by random seed (Appendix C), and by experimental condition (singular/plural target × attractor type)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where models fail: the agreement attraction effect (mismatched attractor conditions), the initial frequency bias phase, and the non-monotonic accuracy drops during training. Smaller models show less stable patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that disaggregation reveals performance decreases on certain conditions during training (e.g., mismatched attractor conditions get worse before they get better), and that smaller models show less stable/reliable patterns."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that disaggregation reveals distinct training phases corresponding to specific heuristics. Figures 1-6 clearly show these phases: frequency bias → local context sensitivity → generalization."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-style claims ('models appear to become sensitive to the preceding word', 'transformers overfit their predictions to token unigram probability, then bigram probability') based on observational analysis of training curves. These are presented as explanations, not rigorously tested causal hypotheses. The paper acknowledges this: 'it may be premature to draw any strong conclusions... without further confirmatory analyses.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states: 'we only investigate language model performance on English subject-verb agreement, and only consider attractors occurring within prepositional phrases' and acknowledges the models used (PolyPythia only). The paper explicitly bounds its scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses whether n-gram-like behavior is the explanation or whether models develop 'a more general ability to make predictions based on an increasingly long context,' notes this 'is a question for future work,' and discusses the construct validity implications of bigram-solvable tasks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on subject-verb agreement and discusses what this does and does not tell us about grammatical knowledge. Section 4 explicitly discusses whether n-gram statistics vs. generalized grammatical rules explain the behavior, acknowledging the gap between the metric and the broader claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies PolyPythia models (van der Wal et al., 2024), which are specific re-releases of Pythia models (Biderman et al., 2023) at sizes 14M to 410M parameters with 10 random seeds each. These are precisely identified model artifacts."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. The paper calculates log-probabilities of verb forms given their context, which is a direct model probability query, not a prompting task."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No training or fine-tuning is performed, and no generation hyperparameters (temperature, etc.) are relevant since the paper only computes log-probabilities."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a direct model evaluation study."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes which dataset subsets are used, how the Bock and Cutting stimuli were preprocessed by Arehalli and Linzen (2020), how simple agreement sentences were created by removing prepositional phrases, and how multi-token verbs are handled (sum vs. normalized log-probabilities)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears between Section 4 (Discussion) and Section 5 (Conclusions), discussing three specific limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies study-specific threats: (1) only English subject-verb agreement with PP attractors, (2) only PolyPythia models because no other suite has the required combination of sizes, seeds, and checkpoints, (3) the work is exploratory and conclusions may be premature without confirmatory analysis."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what was NOT tested: other languages, other attractor types, other model families. It also acknowledges the exploratory nature of the work."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code is released, datasets used are public (BIG-bench, Arehalli and Linzen stimuli), and model checkpoints (PolyPythia) are publicly available. All raw inputs are independently accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 (Datasets) describes exactly which datasets are used, their sources, how stimuli were constructed, and what modifications were made (e.g., removing prepositional phrases to create simple agreement sentences)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks and pre-trained model checkpoints."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 (Procedure) documents the pipeline: calculate log-probability of each verb following its context, compare correct vs. incorrect forms, separate single-token and multi-token verbs, define multi-token probability as sum of token log-probabilities."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'James Michaelov was supported by a grant from the Andrew W. Mellon foundation (#2210-13947) during the writing of this paper.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: James A. Michaelov (MIT), Catherine Arnett (EleutherAI). EleutherAI produced the Pythia/PolyPythia models used in the study."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The Andrew W. Mellon Foundation is a general research funder with no stake in the specific outcomes of this NLP study."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Catherine Arnett is affiliated with EleutherAI which produced the models being evaluated, but no conflict-of-interest disclosure addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff for the Pythia/PolyPythia models, though it references The Pile (Gao et al., 2020) indirectly via the verb frequency table. No explicit cutoff date is given."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the BIG-bench subject-verb agreement stimuli or the Bock and Cutting (1992) stimuli could appear in The Pile training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "BIG-bench was published online and could be in The Pile or its derivatives. The paper does not discuss this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or compute time is reported, despite evaluating 50 model variants (5 sizes × 10 seeds) across many training checkpoints."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget stated. The paper uses pre-trained checkpoints so training cost is not its responsibility, but inference cost across all checkpoints is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "The PolyPythia suite provides 10 random seeds per model size, and Appendix C shows seed-level plots. Section 3 notes 'There is also some variation by random seed (Section C).'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 states: 'These are a set ten random seeds of each Pythia model from 14M to 410M parameters.' The number of seeds (10) per size is explicit."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter tuning is performed. The study evaluates pre-trained models with a fixed evaluation procedure."
       },
       "best_config_selection_justified": {
         "applies": false,
         "answer": false,
         "justification": "No configuration selection is performed. All model sizes and seeds are evaluated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied, despite making many implicit comparisons across conditions, model sizes, and training steps."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not propose a new system to compare against baselines. It analyzes existing models on existing benchmarks."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "Compute budget differences are not relevant — the paper analyzes training dynamics, not proposing a more efficient method."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 explicitly discusses construct validity: 'if a task is solvable based on bigram statistics, it may indicate that the task may not have sufficient construct validity' and discusses the implications of BLiMP subtasks being solvable by 5-grams."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in this study."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether BIG-bench stimuli or the Bock and Cutting stimuli were available during training data collection for The Pile."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The minimal pair design provides both correct and incorrect forms, but the paper does not discuss whether this introduces any evaluation artifacts."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data contains similar subject-verb agreement patterns to the test stimuli (which it almost certainly does, being drawn from English text)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used."
       }
     }
   },
   "claims": [
     {
       "claim": "Aggregated metrics of performance hide interpretable patterns in the trajectory of language models' grammatical knowledge over the course of training.",
       "evidence": "Figure 1 shows that while aggregate accuracy shows slow gradual improvement, condition-level accuracy reveals rapid non-monotonic changes beginning far earlier in training (Section 3-4).",
       "supported": "strong"
     },
     {
       "claim": "Models initially learn to assign higher probability to the more frequent form of the verb (frequency heuristic).",
       "evidence": "Section 3-4: 'is' preferred over 'are' initially (is is more frequent); plural bare forms preferred over singular inflected forms for other verbs. Frequency data from The Pile provided in Table 1 (Appendix A) confirms the frequency ordering matches the observed bias.",
       "supported": "strong"
     },
     {
       "claim": "After the frequency phase, models become sensitive to local context (preceding word), producing agreement attraction effects.",
       "evidence": "Section 3: sharp increase in accuracy for simple plural conditions and conditions with matching attractors, with corresponding decrease for mismatched attractor conditions around steps 128-512 (Figure 1).",
       "supported": "moderate"
     },
     {
       "claim": "These training phases may correspond to models learning unigram, then bigram, then trigram statistics.",
       "evidence": "Section 4 proposes this as a 'possible explanation,' citing Chang et al. (2024) on transformer overfit to n-gram probabilities. The multi-token verb delay (requiring trigram sensitivity) is consistent, but this is explicitly not confirmed.",
       "supported": "weak"
     },
     {
       "claim": "Learning of grammatical rules proceeds in a sequence of 'hidden breakthroughs' rather than being sudden or gradual.",
       "evidence": "Section 5 (Conclusions): the disaggregated results show rapid phase transitions at the condition level that are invisible in aggregate metrics. Supported by Figures 1-6 across model sizes and seeds.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests",
       "detail": "All claims about differences between conditions and training phases are based on visual inspection of plots. No statistical tests are used to confirm that observed differences are significant, despite having 10 seeds to compute statistics."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "BIG-bench subject-verb agreement stimuli were publicly available online and could appear in The Pile training corpus. This is never discussed."
     },
     {
       "flag": "EleutherAI affiliation undisclosed as conflict",
       "detail": "Co-author is affiliated with EleutherAI, which produced the Pythia/PolyPythia models being evaluated. While this is disclosed as an affiliation, it is not discussed as a potential conflict of interest."
     }
   ],
   "cited_papers": [
     {
       "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
       "authors": [
         "Aarohi Srivastava et al."
       ],
       "year": 2023,
       "relevance": "BIG-bench benchmark used as the primary dataset; major LLM capability evaluation suite."
     },
     {
       "title": "Emergent Abilities of Large Language Models",
       "authors": [
         "Jason Wei",
         "Yi Tay",
         "Rishi Bommasani"
       ],
       "year": 2022,
       "relevance": "Key paper on emergent abilities debate that this paper directly engages with."
     },
     {
       "title": "Are Emergent Abilities of Large Language Models a Mirage?",
       "authors": [
         "Rylan Schaeffer",
         "Brando Miranda",
         "Sanmi Koyejo"
       ],
       "year": 2023,
       "relevance": "Counter-argument on emergent abilities; this paper provides new evidence relevant to the sudden vs. gradual learning debate."
     },
     {
       "title": "Embers of autoregression show how large language models are shaped by the problem they are trained to solve",
       "authors": [
         "R. Thomas McCoy",
         "Shunyu Yao",
         "Dan Friedman"
       ],
       "year": 2024,
       "relevance": "Argues LLMs are shaped by surface-level heuristics; directly relevant to this paper's findings about frequency and n-gram heuristics."
     },
     {
       "title": "Dissociating language and thought in large language models",
       "authors": [
         "Kyle Mahowald",
         "Anna A. Ivanova",
         "Idan A. Blank"
       ],
       "year": 2024,
       "relevance": "Argues contemporary LLMs show linguistic competence; this paper investigates how that competence is acquired."
     },
     {
       "title": "Can Language Models Handle Recursively Nested Grammatical Structures? A Case Study on Comparing Models and Humans",
       "authors": [
         "Andrew Lampinen"
       ],
       "year": 2024,
       "relevance": "Shows even large models like Chinchilla fail at difficult grammatical tasks; directly motivates this study."
     },
     {
       "title": "PolyPythias: Stability and Outliers across Fifty Language Model Pre-Training Runs",
       "authors": [
         "Oskar van der Wal"
       ],
       "year": 2024,
       "relevance": "Provides the PolyPythia model suite used in this study; relevant to understanding training stability and reproducibility."
     },
     {
       "title": "Hidden Breakthroughs in Language Model Training",
       "authors": [
         "Sophia Kangaslahti",
         "Elan Rosenfeld",
         "Naomi Saphra"
       ],
       "year": 2025,
       "arxiv_id": "2506.15872",
       "relevance": "Introduces the 'hidden breakthroughs' concept that this paper's findings support and extend."
     },
     {
       "title": "Characterizing Learning Curves During Language Model Pre-Training: Learning, Forgetting, and Stability",
       "authors": [
         "Tyler A. Chang",
         "Zhuowen Tu",
         "Benjamin K. Bergen"
       ],
       "year": 2024,
       "relevance": "Documents n-gram overfitting progression during training, providing a possible mechanistic explanation for this paper's findings."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The disaggregation methodology could inform how practitioners evaluate LM training, but requires significant adaptation beyond this narrow grammatical domain."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The main finding that smooth aggregate learning curves hide rapid non-monotonic phase transitions is counterintuitive and challenges the gradual-vs-sudden learning debate."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk angle whatsoever."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no company challenges, purely academic contribution to an ongoing scientific debate."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires setting up PolyPythia models and running evaluation scripts, not a quick try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "MIT and EleutherAI are recognized in ML circles but not household names; NeurIPS venue adds credibility but the topic is niche."
     }
   }
 }