ai-research-survey

scan.json (31361B)

---

 {
   "paper": {
     "title": "Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models",
     "authors": [
       "Aarohi Srivastava",
       "Abhinav Rastogi",
       "Abhishek Rao",
       "Abu Awal Md Shoeb",
       "Abubakar Abid",
       "Adam Fisch",
       "Adam R. Brown",
       "Adam Santoro",
       "Aditya Gupta",
       "Adrià Garriga-Alonso"
     ],
     "year": 2022,
     "venue": "Transactions on Machine Learning Research (TMLR)",
     "arxiv_id": "2206.04615"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The BIG-bench benchmark is released as a GitHub repository, referenced throughout (Section 2: 'The Beyond the Imitation Game benchmark (BIG-bench) GitHub repository includes: ... Code that implements the benchmark API'). The benchmark code, task definitions, and evaluation infrastructure are publicly available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The BIG-bench benchmark data (204 tasks) is released via the GitHub repository. Section 2 states it includes task definitions, evaluation results, and 'analysis Colab notebooks, as well as data files containing scores and logs of model interactions on each task' (Section 6.1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes a Python-based API (Section 2.1) and references Colab notebooks, no requirements.txt, Dockerfile, or detailed environment setup with library versions is provided in the paper itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes the BIG-bench API in detail, including how tasks interact with models, the two task types (JSON and programmatic), and the available methods (generate_text, cond_log_prob). The GitHub repository is referenced for 'instructions on how to contribute' and evaluate models. Section 6.1 mentions release of 'analysis Colab notebooks.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 caption states: 'Error bars are obtained by 10-bootstrap sampling over evaluated tasks.' Bootstrap error bars are shown for calibration metrics across model scales."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., sparse models outperform dense, bias increases with scale) but does not report any statistical significance tests such as p-values, t-tests, or confidence intervals for these comparisons. Differences are presented by visual comparison of curves."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides quantitative effect sizes with context. For example, Section 3.3 states sparse models achieve 'a roughly twofold improvement in inference cost' and 'about a tenfold improvement in the FLOP-matched parameter count needed to reach a given calibration score.' Section 3.6 states the 128B model finds it 'over 22 times more likely that a white boy will grow up to be a good doctor than that a Native American girl will.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark consists of 204 tasks contributed by 450 authors, but there is no explicit justification for why this number of tasks was chosen, no power analysis, and no discussion of whether this sample is sufficient for the aggregate analyses performed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are generally reported as single runs per model per task. While bootstrap error bars appear in Figure 5 (across tasks, not across runs), there is no reporting of variance across experimental runs, seeds, or repeated evaluations of the same model on the same tasks."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human expert rater baselines are provided (Section 2.3.2): 'a team of human expert raters performed all tasks in order to provide a strong baseline.' Both mean and max human rater scores are reported. Random chance baselines are also shown on multiple figures."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates multiple contemporary model families: GPT-3 series (Brown et al., 2020), Google-internal BIG-G models, BIG-G sparse models, and PaLM (Chowdhery et al., 2022). These were state-of-the-art models at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "While not a traditional ablation, the paper systematically varies model size across six orders of magnitude, compares dense vs. sparse architectures, and examines the effect of shot count (0-3 shot). Section 3.5 investigates sensitivity to task formatting (with/without multiple choice options, different phrasings of cause_and_effect)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: exact_string_match, multiple_choice_grade, expected_calibration_error, multiple_choice_brier_score, BLEU, BLEURT, ROUGE, case_insensitive_str_match, and normalized preferred metrics (Section 2.1). Section 3.4.2 explicitly discusses how different metrics reveal different patterns."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3.2 describes human expert raters who completed all tasks: 'we employed a team of expert raters to complete the tasks submitted to BIG-bench.' Mean and max human scores are reported. Expert evaluation is further discussed in Appendix E."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.4 describes the canary string mechanism to prevent data contamination, and Section 2.3.1 states: 'The data used to train all models (except PaLM) was collected before the BIG-bench repository was created. Direct leakage of BIG-bench tasks into models reported in this paper is therefore impossible.' The benchmark tasks serve as held-out test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-task breakdowns are provided: Figure 4 shows per-task BIG-bench Lite performance, Figure 7 shows tasks grouped by linearity and breakthroughness scores, Appendix D shows performance by keyword, and Section 3.6 breaks down bias results by category (gender, religion, race/ethnicity, nationality)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 ('Even large language models are brittle') extensively discusses failure cases. Section 4.1 analyzes chess failures qualitatively. Section 4.2 describes how models output nonsensical text at small scales. Section 3.7 discusses failures on non-English and low-resource languages."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results are reported: social bias increases with scale (Section 3.6), models remain far below human performance (Section 3.1), performance on low-resource languages fails to improve with scale (Section 3.7), models are brittle to phrasing changes (Section 3.5), and tasks with lowest linearity show performance degrading with scale (Figure 7c)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims are well-supported: (1) model performance improves with scale but is poor in absolute terms (Figure 1, Section 3.1), (2) calibration improves with scale (Figure 5, Section 3.2), (3) performance is similar across model classes with benefits from sparsity (Figure 6, Section 3.3), (4) breakthrough vs. gradual scaling behavior (Figure 7, Section 3.4), (5) social bias increases with scale in ambiguous contexts but can be improved with prompting (Figures 12-13, Section 3.6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper is generally careful about causal language. It uses hedged language like 'One possible explanation for the breakthrough phenomenon' (Section 6.1) and 'We speculate that models perform poorly on the other versions because...' (Section 3.5). The systematic variation of model size while holding other factors constant provides reasonable support for scale-related claims. Ablation-like comparisons (dense vs. sparse, with/without choice options) use controlled manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its generalizations. Section 2.3.2 cautions about interpreting human baselines. Section 3.4.1 states observations are 'anecdotal.' Section 6.1 identifies specific limitations 'that we believe will require new approaches.' The Broader Impact Statement acknowledges BIG-bench 'certainly has holes in what is covered.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4.2 discusses alternative explanations for breakthrough behavior (metric artifacts vs. genuine capability jumps). Section 3.5 considers that models may perform poorly on reformulated tasks because those are 'dissimilar from their training distribution.' Section 3.7 considers whether Swahili performance indicates 'general understanding of Swahili or instead memorization of proverbs.' Section 4.2 considers training data recency as explanation for element naming errors."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are described by architecture family and parameter count (e.g., 'BIG-G' with 13 sizes, 'GPT' corresponding to 'GPT-3 model series in Brown et al. (2020)'). Appendix A provides detailed architecture hyperparameters. However, specific model version identifiers, snapshot dates, or API versions are not provided. The GPT models are referenced generically as 'OpenAI GPT models corresponding to the GPT-3 model series.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt examples are provided for multiple tasks throughout the paper: cause_and_effect task formats (Section 3.5), checkmate_in_one (Section 4.1), periodic_elements (Section 4.2), simple_arithmetic_json_multiple_choice (Section 3.5), and emoji_movie (Figure 9c). The BIG-bench repository contains all task definitions with their prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3.1 states: 'All model outputs were sampled greedily (with zero temperature), unless otherwise noted.' Temperature=1, top-k=40 is also tested (Figure App.1). Appendix A (Tables App.1 and App.2) provides detailed model architecture hyperparameters including layers, dimensions, training steps, and warmup steps."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct prompting with the BIG-bench API (generate_text, cond_log_prob methods). Some programmatic tasks involve multiple query rounds, but this is part of the benchmark specification, not an agentic scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 documents the task types (JSON and programmatic), how performance is evaluated (comparing generated outputs to targets using standard metrics), and the scoring normalization procedure (Section 3.1). Section 2.2 describes the BBL selection process. Section 2.3.1 describes sampling procedures. The normalized preferred metric calculation is explicitly defined."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Broader Impact Statement (after Section 6.2) discusses limitations extensively: 'BIG-bench certainly has holes in what is covered' and discusses gaps in coverage. Section 6.1 explicitly lists limitations requiring new approaches. Section 2.3.2 has a detailed discussion of limitations of human baseline interpretation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Multiple specific threats are discussed: Section 3.4.3 warns that 'capabilities and trajectories across scale of language models are much more subjective than we would like to believe.' Section 2.3.2 notes human rater scores are confounded by time controls, task changes, and non-representative demographics. Section 6.1 identifies specific capability gaps (long contexts, episodic memory, recurrent computation, multi-modal grounding)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Broader Impact Statement states: 'BIG-bench certainly has holes in what is covered' and that 'we have explicitly limited BIG-bench's scope' regarding domains beyond text. Section 2.1 acknowledges 'BIG-bench is designed for evaluating pure language models and has limits to cover multi-modal capabilities.' Section 6.1 lists specific things models cannot do that BIG-bench may not fully capture."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 states they 'release analysis Colab notebooks, as well as data files containing scores and logs of model interactions on each task.' The benchmark tasks and all evaluation results are available in the GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.3 describes the collection process: 'The benchmark was developed openly on GitHub, with contributors adding tasks by way of GitHub pull requests. Peer review of the proposed tasks was performed by discussion on the pull request.' Section 7 provides detailed author contributions. The review criteria for task acceptance are referenced."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how task contributors were recruited: through an open call on GitHub (Section 1.3), with authorship offered as incentive. Section 7 details core contributor roles. For human raters, Section 2.3.2 and Appendix E describe the evaluation team. However, the demographic representativeness of raters is flagged as a limitation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from task submission through peer review to benchmark inclusion is described (Section 1.3). The evaluation pipeline is documented: model querying via API → metric computation → normalization (Section 3.1). The filtering/debugging of tasks post-merge is described in Section 7.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section or grant acknowledgments appear in the paper. The Acknowledgments section thanks individuals for discussion and feedback but does not mention any funding sources. Given the scale of the work and involvement of Google and OpenAI employees, funding disclosure would be expected."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are provided throughout Section 7.2 for each task contribution, identifying institutions including Google, OpenAI, and many universities. Section 7.1 identifies core contributors and their affiliations. The involvement of Google and OpenAI employees in evaluating their own models is implicit from the affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google employees trained the BIG-G models and are core contributors to the benchmark (Section 7.1). OpenAI employees contributed GPT evaluations. Both companies have commercial interest in language model capabilities. No discussion of whether funders are independent of outcomes is provided. Since no funding is disclosed but Google and OpenAI resources were clearly used, the funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper. Given that multiple authors are employees of Google and OpenAI — companies with direct commercial interest in language model capabilities — this is a notable omission."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3.1 states: 'The data used to train all models (except PaLM) was collected before the BIG-bench repository was created. Direct leakage of BIG-bench tasks into models reported in this paper is therefore impossible.' While not giving exact cutoff dates, the temporal relationship between training data and benchmark creation is clearly established."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3.1 explicitly addresses this: 'Direct leakage of BIG-bench tasks into models reported in this paper is therefore impossible. Indirect leakage is possible, since many tasks use text available on the internet.' Section 2.4 describes the canary string mechanism for filtering BIG-bench data from future training corpora. The training_on_test_set task provides post-hoc diagnosis."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple contamination countermeasures are described: (1) temporal separation — models trained before benchmark creation (Section 2.3.1), (2) canary string GUID for corpus filtering (Section 2.4), (3) training_on_test_set task for post-hoc detection (Section 2.4), (4) acknowledgment that 'indirect leakage is possible' with mitigation strategies."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No pre-registration is mentioned for the human evaluation study. The benchmark was developed as an open collaboration, but the specific evaluation protocol with human raters was not pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned despite employing human raters to perform benchmark tasks over a period of weeks."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.3.2 acknowledges that 'The demographics and backgrounds of the expert raters are not necessarily representative of the population in general' but does not actually report demographic information about the raters beyond listing their names (Section 7.1)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria for human raters are stated. The paper lists the names of raters (Section 7.1) and mentions they include core authors and an 'expert evaluation team' but does not describe how they were selected or what qualifications were required."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study comparing human conditions; humans serve as a baseline. There are no treatment/control groups requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Blinding is not applicable here — humans performed benchmark tasks as a baseline comparison, not as evaluators of model outputs in a blinded experiment."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 2.3.2 mentions that tasks were subsampled and performed over weeks with time controls, but does not report how many raters started vs. finished, which raters completed which tasks, or any attrition information."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite evaluating models across six orders of magnitude of scale on 204 tasks, no inference cost, latency, or computational cost per evaluation is reported. Section 2.2 acknowledges 'full evaluation is computationally expensive' but does not quantify this."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated for the evaluation or for training the BIG-G models. Appendix A provides model architecture details and training steps but not GPU hours, wall-clock time, or total compute cost."
       }
     }
   },
   "claims": [
     {
       "claim": "Model performance and calibration both improve with scale, but models perform poorly in absolute terms compared with human raters.",
       "evidence": "Figure 1 shows aggregate performance across all models. Section 3.1: 'even the strongest models perform poorly overall when compared with expert human rater performance.' The best performing language models achieved a normalized preferred metric score of less than 20 out of 100. Figure 5 shows calibration improves with scale.",
       "supported": "strong"
     },
     {
       "claim": "Performance is remarkably similar across model classes (GPT, BIG-G), with benefits from sparsity.",
       "evidence": "Section 3.3 and Figure 6: 'BIG-G sparse models perform better on BIG-bench tasks than BIG-G dense models, achieving a roughly twofold improvement in inference cost.' Sparse models show 'about a tenfold improvement in the FLOP-matched parameter count needed to reach a given calibration score.'",
       "supported": "strong"
     },
     {
       "claim": "Tasks that improve gradually commonly involve knowledge/memorization, while breakthrough tasks involve multiple steps or components, or brittle metrics.",
       "evidence": "Section 3.4.1: highest linearity tasks are 'knowledge-based' (qa_wikidata, linguistic_mappings). Breakthrough tasks are 'composite in nature' (modified_arithmetic, codenames). Section 3.4.2 shows metrics can create apparent breakthroughs. Figure 7 illustrates both categories.",
       "supported": "moderate"
     },
     {
       "claim": "Social bias typically increases with scale in ambiguous contexts, but can be improved with prompting.",
       "evidence": "Section 3.6 and Figure 12 show aggregate bias increasing with scale across six tasks. Figure 13 shows bias decreasing with scale in unambiguous contexts and with pro-social prompts. The 128B model finds it 'over 22 times more likely that a white boy will grow up to be a good doctor than that a Native American girl will.'",
       "supported": "strong"
     },
     {
       "claim": "Breakthrough behavior is often an artifact of metric choice rather than a genuine discontinuity in capability.",
       "evidence": "Section 3.4.2 demonstrates that smoother metrics (BLEU, log probability) reveal gradual progress hidden by exact_string_match. Figure 8 shows log probabilities improve smoothly for breakthrough tasks. Figure 9 shows emoji_movie appears gradual with multiple_choice_grade but abrupt with exact_string_match.",
       "supported": "strong"
     },
     {
       "claim": "Large language models are brittle — sensitive to precise phrasing in unintuitive ways.",
       "evidence": "Section 3.5: Figure 10 shows models perform substantially better on multiple-choice questions when choices are NOT included in the input. Figure 11 shows cause_and_effect performance varies dramatically across three task formulations (from chance to near-perfect).",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "BIG-bench introduces a 204-task benchmark evaluated across dense and sparse language models spanning six orders of magnitude. Model performance improves with scale but remains far below human expert baselines (best model scores below 20/100 normalized). Scaling behavior is task-dependent: knowledge-based tasks improve linearly while composite tasks show breakthrough behavior, though this is often a metric artifact rather than a genuine capability discontinuity. Social bias generally increases with model scale in ambiguous contexts but can be mitigated through prompting, and sparse models provide roughly 2x efficiency gains over dense models at equivalent performance.",
   "red_flags": [
     {
       "flag": "Company evaluating own models",
       "detail": "Google employees trained the BIG-G models and are core contributors to the benchmark. OpenAI employees contributed GPT evaluation code. Both companies have commercial interest in demonstrating language model capabilities, yet no conflict of interest statement appears in the paper."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Despite numerous comparative claims about model performance differences (dense vs. sparse, across model sizes, bias trends), no statistical significance tests are reported. Conclusions are drawn from visual inspection of curves."
     },
     {
       "flag": "Human baseline limitations",
       "detail": "The human evaluation baseline has significant confounders acknowledged but not fully addressed: time controls (30 min to 2 hours per session), task subsampling, non-representative demographics, and some raters were paper co-authors. These factors likely underestimate human performance, making the human-model gap comparison less reliable."
     },
     {
       "flag": "No compute costs reported",
       "detail": "Evaluating 204 tasks across models spanning six orders of magnitude is computationally expensive (acknowledged in Section 2.2), but no compute budget, inference costs, or training costs are reported. This hinders reproducibility assessments."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Foundational GPT-3 paper; the GPT models evaluated in BIG-bench correspond to this model series."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval paper demonstrating LLM code generation capabilities, directly relevant to code generation benchmarking."
     },
     {
       "title": "PaLM: Scaling Language Modeling with Pathways",
       "authors": ["Aakanksha Chowdhery"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "PaLM evaluation on BIG-bench Lite is included in the paper; demonstrates state-of-the-art model performance and reduced brittleness at scale."
     },
     {
       "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?",
       "authors": ["Emily M. Bender"],
       "year": 2021,
       "relevance": "Discusses socially harmful effects of large language models, directly relevant to the survey's coverage of AI safety and bias."
     },
     {
       "title": "On the Opportunities and Risks of Foundation Models",
       "authors": ["Rishi Bommasani"],
       "year": 2021,
       "arxiv_id": "2108.07258",
       "relevance": "Comprehensive report on foundation models covering capabilities, applications, and societal impact — key context for understanding LLM evaluation landscape."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Establishes scaling laws for language model performance, foundational to BIG-bench's analysis of how capabilities change with model size."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks"],
       "year": 2021,
       "arxiv_id": "2009.03300",
       "relevance": "MMLU benchmark with 57 tasks; a contemporary benchmark that BIG-bench aims to complement with greater diversity and difficulty."
     },
     {
       "title": "Predictability and Surprise in Large Generative Models",
       "authors": ["Deep Ganguli"],
       "year": 2022,
       "arxiv_id": "2202.07785",
       "relevance": "Discusses breakthrough/emergent capabilities in language models, directly motivating BIG-bench's study of scaling behavior and breakthroughness."
     },
     {
       "title": "Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity",
       "authors": ["William Fedus"],
       "year": 2021,
       "arxiv_id": "2101.03961",
       "relevance": "Introduces the sparse transformer architecture evaluated in BIG-bench as BIG-G sparse, relevant to efficient model scaling."
     },
     {
       "title": "Training Compute-Optimal Large Language Models",
       "authors": ["Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling analysis suggesting optimal compute allocation; evaluated on BIG-bench and informing discussion of scaling trends."
     },
     {
       "title": "DynaBench: Rethinking Benchmarking in NLP",
       "authors": ["Douwe Kiela"],
       "year": 2021,
       "arxiv_id": "2104.14337",
       "relevance": "Dynamic benchmarking platform using human-and-model-in-the-loop; complementary approach to BIG-bench's static benchmark design."
     },
     {
       "title": "LaMDA: Language Models for Dialog Applications",
       "authors": ["Romal Thoppilan"],
       "year": 2022,
       "arxiv_id": "2201.08239",
       "relevance": "BIG-G models are based on LaMDA architectures; the paper discusses LaMDA's safety techniques as important for future models."
     }
   ]
 }