ai-research-survey

scan.json (28000B)

---

 {
   "paper": {
     "title": "Are Emergent Abilities of Large Language Models a Mirage?",
     "authors": ["Rylan Schaeffer", "Brando Miranda", "Sanmi Koyejo"],
     "year": 2023,
     "venue": "NeurIPS 2023",
     "arxiv_id": "2304.15004"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage", "survey_methodology"],
   "methodology_tags": ["meta-analysis", "benchmark-eval", "theoretical"],
   "key_findings": "The paper demonstrates that claimed emergent abilities of LLMs are likely artifacts of metric choice rather than fundamental phase transitions in model behavior. Over 92% of claimed emergent abilities on BIG-Bench appear under just two metrics (Multiple Choice Grade and Exact String Match), both of which nonlinearly or discontinuously scale per-token error rates. Switching to linear or continuous metrics (Token Edit Distance, Brier Score) reveals smooth, predictable performance improvement. The authors further show that seemingly emergent abilities can be artificially induced in vision tasks by choosing appropriately sharp metrics.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is mentioned in the paper. The paper is listed as 'Preprint. Under review' with no link to reproduction code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available data: BIG-Bench is public, GPT-3/InstructGPT outputs were collected via a public API, and the hand-annotated emergent abilities list from Wei (2022) is public. CIFAR100, MNIST, and Omniglot are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, requirements files, or dependency details are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The experimental setup is described in prose but no scripts or detailed reproduction steps are given."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports point estimates for metrics across model scales. No confidence intervals or error bars are shown on the main figures (Figs. 2-8)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., emergent abilities disappear with different metrics) but does not use any statistical significance tests. Comparisons are made visually via plots."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concrete effect sizes: '>92% of emergent abilities on BIG-Bench appear under either of these two metrics' (Sec. 1), and provides quantitative breakdowns of which metrics produce emergence across all 39 BIG-Bench metrics (Fig. 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of tasks analyzed, the number of test examples generated for GPT-3 experiments, or the model family sizes chosen for vision experiments."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across experimental runs for any of the three analysis tracks (GPT-3, BIG-Bench meta-analysis, vision experiments)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly compares against the original emergent abilities claims (Wei et al. 2022, Ganguli et al. 2022, Srivastava et al. 2022), using the same model families and tasks but with different metrics."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are the original emergent abilities papers (2022), which are the most recent and relevant prior work on this topic."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The core methodology is effectively an ablation: holding model outputs fixed while varying the metric (Accuracy vs Token Edit Distance, Multiple Choice Grade vs Brier Score) to isolate the metric's contribution to the emergence phenomenon."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly uses multiple metrics (Accuracy, Token Edit Distance, Multiple Choice Grade, Brier Score, cross-entropy, Reconstruction_c) and shows how results change across them."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to the claims, which are about mathematical properties of metrics applied to model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For GPT-3 experiments, the authors generate new test data for arithmetic tasks. For BIG-Bench, standard test splits are used. For vision experiments, standard test sets (CIFAR100, MNIST, Omniglot) are used."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 5 provides per-metric breakdowns across all 39 BIG-Bench metrics. Figs. 3-4 break down by target string length (1-5 digits). Fig. 6 shows per-task LaMDA results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss cases where changing the metric did NOT remove the emergent ability, or tasks where the alternative explanation might not fully apply."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every experiment confirms the authors' hypothesis. No failed attempts, counterexamples, or cases where the alternative explanation was insufficient are reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that emergent abilities appear due to metric choice, and the paper provides three lines of evidence: GPT-3 arithmetic (Sec. 3), BIG-Bench meta-analysis (Sec. 4), and induced emergence in vision (Sec. 5), all supporting this claim."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims metric choice 'creates' emergent abilities. This causal claim is justified by a controlled manipulation: holding model outputs fixed and varying only the metric, which is a valid single-variable manipulation. The mathematical model (Sec. 2) provides the theoretical grounding."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion (Sec. 7) explicitly states: 'nothing in this paper should be interpreted as claiming that large language models cannot display emergent abilities; rather, our message is that previously claimed emergent abilities in [3, 8, 28, 33] might likely be a mirage.' The claims are bounded to the specific prior work analyzed."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses alternative explanations: Caballero et al.'s piece-wise power law model (where emergence is real), Michaud et al.'s strong data assumptions, and Srivastava et al.'s original hypothesis. The paper positions itself against these alternatives."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is about the distinction between what metrics measure (per-token error rate) and what researchers claim they show (emergent abilities). The paper explicitly argues that the proxy (discontinuous/nonlinear metric) distorts the underlying phenomenon (smooth improvement)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '4 models with 350M, 1.3B, 6.7B, 175B parameters are available via the OpenAI API' (footnote 3) but does not specify exact model version identifiers or API snapshot dates. The LaMDA analysis uses published BIG-Bench outputs."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions '2-shot multiplication between two 2-digit integers and 2-shot addition between two 4-digit integers' but does not provide the actual prompt text used to query the GPT-3 models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Fig. 3 shows results at temperature 0.0 and 1.0 but no other API parameters (top-p, max tokens) are reported. Vision experiment hyperparameters (learning rates, optimizers, training epochs) are not reported in the main text."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper queries models directly and analyzes outputs."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "For GPT-3 experiments, how arithmetic test examples were generated is not described. For the BIG-Bench meta-analysis, the filtering from all tasks to those with claimed emergent abilities is described but the exact criteria for the emergence score threshold used in Fig. 5A are not stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The Discussion (Sec. 7) includes one caveat ('nothing in this paper should be interpreted as claiming that large language models cannot display emergent abilities') but no structured limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. For example, the paper does not address whether its mathematical model's independence assumption (footnote 1: 'the independence assumption is not true') might affect the conclusions, or whether the GPT-3 analysis with only 4 model sizes is sufficient."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion states specific scope boundaries: 'nothing in this paper should be interpreted as claiming that large language models cannot display emergent abilities' and that the claims are specifically about 'previously claimed emergent abilities in [3, 8, 28, 33].'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The GPT-3 model outputs collected by the authors are not released. BIG-Bench data is public but the authors' specific analysis scripts and intermediate data are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes collecting outputs from InstructGPT/GPT-3 via the OpenAI API on arithmetic tasks. Section 4 describes using BIG-Bench published outputs and hand-annotated emergence data from Wei (2022). Section 5 describes training vision models on standard datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks and public APIs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw BIG-Bench outputs to the emergence score analysis is not fully documented. How many task-metric-model family triplets were analyzed, how many were excluded, and the exact filtering steps are not stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned in the paper. There is no acknowledgments section listing grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: all three authors are from Computer Science, Stanford University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GPT-3/InstructGPT models but does not state their training data cutoff dates. For vision models trained by the authors, training data is standard benchmarks so cutoff is less relevant, but for the LLM evaluation it matters."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-3 may have seen arithmetic problems similar to the test set in its training data. The arithmetic tasks are generated, but the format may overlap with training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "BIG-Bench tasks were publicly available before GPT-3's training. No contamination analysis is performed. However, contamination would actually strengthen the paper's argument (if models saw answers, smooth improvement would be even more expected), so this is a lesser concern."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper queries GPT-3/InstructGPT via API and trains vision models but does not report API costs or compute costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated for any of the three experimental tracks."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis for vision experiments (autoencoders, LeNet, transformers). GPT-3 API queries may have used temperature but no multi-seed analysis is shown."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated for any experiment. It is unclear whether vision models were trained once or multiple times."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the vision experiments. The mathematical model parameters (c, α) are not described as searched."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how hyperparameters for vision experiments were selected or whether the shown configurations were cherry-picked from a larger set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable. Interestingly, the paper notes the multiple comparisons problem in BIG-Bench (Sec. 7: '~10^6 task-metric-model family triplets') but as a criticism of prior work, not as something requiring correction in their own analysis."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors define their own metrics (Token Edit Distance, Reconstruction_c) and their own mathematical model to test their hypothesis. They do not acknowledge that their choices of alternative metrics and model parameters could be tuned to support their thesis."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "The paper compares metrics at fixed model scales, not compute budgets. Compute differences are not the focus."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is a construct validity critique: it argues that benchmarks using discontinuous/nonlinear metrics do not validly measure emergence because the metric creates the appearance of a phase transition. This is the core contribution."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in any of the experiments."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether BIG-Bench tasks or arithmetic formats appeared in GPT-3's training data before the evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 2-shot examples provide leakage or whether the evaluation format matches training distribution."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between training and test data for any of the evaluated models."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "The meta-analysis of BIG-Bench uses a quantitative emergence score (Eq. 1, from Srivastava et al. 2022) but does not follow PRISMA or any structured review protocol for selecting which emergent abilities claims to analyze."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of the source papers claiming emergent abilities. It takes the claims at face value and re-analyzes the metrics, but does not evaluate whether the original experiments were well-designed."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses a form of publication bias: 'emergent abilities claims are possibly infected by a failure to control for multiple comparisons. In BIG-Bench alone, there are ≥220 tasks, ~40 metrics per task, ~10 model families, for a total of ~10^6 task-metric-model family triplets.'"
       }
     }
   },
   "claims": [
     {
       "claim": "Emergent abilities of LLMs are created by the researcher's choice of nonlinear or discontinuous metrics, not fundamental changes in model behavior with scale.",
       "evidence": "Mathematical model (Sec. 2) shows how power-law cross-entropy decline produces sharp transitions under Accuracy but smooth transitions under Token Edit Distance. Three empirical analyses confirm this: GPT-3 arithmetic (Sec. 3), BIG-Bench meta-analysis (Sec. 4), and induced emergence in vision (Sec. 5).",
       "supported": "strong"
     },
     {
       "claim": "Over 92% of claimed emergent abilities on BIG-Bench appear under just two metrics: Multiple Choice Grade and Exact String Match.",
       "evidence": "Fig. 5C shows hand-annotated data from Wei (2022) revealing that 2 out of 39 BIG-Bench metrics account for >92% of emergent abilities. Multiple Choice Grade is discontinuous and Exact String Match is nonlinear.",
       "supported": "strong"
     },
     {
       "claim": "Changing from nonlinear/discontinuous metrics to linear/continuous metrics removes emergent abilities in GPT-3 arithmetic tasks.",
       "evidence": "Fig. 3 shows that switching from Accuracy to Token Edit Distance on 2-digit multiplication and 4-digit addition tasks reveals smooth, predictable improvement. Fig. 4 shows that even under Accuracy, increasing test dataset size reveals above-chance performance in smaller models.",
       "supported": "strong"
     },
     {
       "claim": "LaMDA's emergent abilities on BIG-Bench disappear when measured with Brier Score instead of Multiple Choice Grade.",
       "evidence": "Fig. 6 shows LaMDA results on multiple tasks where sharp transitions under Multiple Choice Grade become smooth curves under Brier Score.",
       "supported": "strong"
     },
     {
       "claim": "Seemingly emergent abilities can be induced in vision tasks (autoencoders, CNNs, transformers) by choosing appropriately sharp metrics.",
       "evidence": "Fig. 7 shows induced emergence in shallow autoencoders on CIFAR100 via Reconstruction_c metric. Fig. 8 shows induced emergence in autoregressive transformers on Omniglot. Fig. 10 shows induced emergence in LeNet on MNIST.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "All experiments confirm hypothesis",
       "detail": "Every single experiment and analysis in the paper supports the authors' alternative explanation. No counterexamples, failed predictions, or boundary cases where the explanation is insufficient are discussed. This uniformly positive pattern is unusual for a paper making such a strong claim."
     },
     {
       "flag": "Limited model sampling",
       "detail": "The GPT-3/InstructGPT analysis uses only 4 model sizes (350M, 1.3B, 6.7B, 175B). With so few data points, visual assessment of 'smooth vs sharp' transitions is subjective. No statistical test is applied to distinguish the two patterns."
     },
     {
       "flag": "Metric choice for alternative may be optimized",
       "detail": "The authors choose Token Edit Distance and Brier Score as their alternative metrics, and define the novel Reconstruction_c metric with specific threshold values. These choices could themselves be optimized to support the thesis, mirroring the very concern they raise about the original emergence claims."
     }
   ],
   "cited_papers": [
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani", "Colin Raffel", "Barret Zoph"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "The primary paper being critiqued — defines and catalogs emergent abilities of LLMs."
     },
     {
       "title": "Beyond the imitation game: Quantifying and extrapolating the capabilities of language models",
       "authors": ["Aarohi Srivastava", "Abhinav Rastogi"],
       "year": 2022,
       "arxiv_id": "2206.04615",
       "relevance": "BIG-Bench benchmark used as the primary data source for the meta-analysis of emergent abilities."
     },
     {
       "title": "Predictability and surprise in large generative models",
       "authors": ["Deep Ganguli", "Danny Hernandez"],
       "year": 2022,
       "relevance": "Key prior work that emphasized emergent abilities as unpredictable and surprising, directly challenged by this paper."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper — the model family used in the primary experimental validation of the alternative explanation."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational neural scaling laws work that underpins the paper's mathematical model of smooth performance improvement."
     },
     {
       "title": "Broken neural scaling laws",
       "authors": ["Ethan Caballero", "Kshitij Gupta", "Irina Rish", "David Krueger"],
       "year": 2022,
       "arxiv_id": "2210.14891",
       "relevance": "Alternative explanation for emergence via piece-wise power laws, contrasted with this paper's metric-based explanation."
     },
     {
       "title": "The quantization model of neural scaling",
       "authors": ["Eric J. Michaud", "Ziming Liu", "Uzay Girit", "Max Tegmark"],
       "year": 2023,
       "relevance": "Posits that emergent abilities may be real under strong data assumptions, an alternative view to this paper's metric artifact explanation."
     },
     {
       "title": "Data distributional properties drive emergent in-context learning in transformers",
       "authors": ["Stephanie CY Chan", "Adam Santoro"],
       "year": 2022,
       "relevance": "Studies emergence in in-context learning with controlled transformer experiments, methodology inspiration for this paper's vision experiments."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws — foundational work on compute-optimal training that informs the scaling assumptions."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "One of the key model families claimed to exhibit emergent abilities, cited as evidence in the original emergence papers."
     },
     {
       "title": "Attention is all you need",
       "authors": ["Ashish Vaswani", "Noam Shazeer"],
       "year": 2017,
       "relevance": "Transformer architecture used in the paper's induced emergence experiments with autoregressive models."
     }
   ]
 }