ai-research-survey

scan.json (30591B)

---

 {
   "paper": {
     "title": "PaLM: Scaling Language Modeling with Pathways",
     "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin", "Maarten Bosma", "Gaurav Mishra", "Adam Roberts", "Paul Barham", "Hyung Won Chung", "Charles Sutton", "Sebastian Gehrmann"],
     "year": 2022,
     "venue": "Journal of Machine Learning Research",
     "arxiv_id": "2204.02311"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PaLM, a 540B parameter dense Transformer trained on 780B tokens using Pathways across 6144 TPU v4 chips, achieves state-of-the-art few-shot results on 28/29 English NLP benchmarks, outperforms average human performance on BIG-bench (150+ tasks), and matches or exceeds finetuned SOTA on reasoning tasks when combined with chain-of-thought prompting. The paper documents discontinuous improvements from scale on ~25% of BIG-bench tasks, demonstrates strong multilingual and code capabilities despite limited non-English training data (~22%), and provides comprehensive analysis of memorization, data contamination, bias, and toxicity.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or model weights were released. The paper references internal infrastructure (JAX, T5X, Pathways) but provides no public repository for reproducing the experiments."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training dataset is not released. It is described as based on LaMDA and GLaM datasets with modifications. Evaluation uses public benchmarks (BIG-bench, SuperGLUE, etc.) which are available, but the training data itself is proprietary."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states training used JAX, T5X, and TPU v4 chips, but provides no dependency versions, requirements files, or sufficient detail to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The training setup is described at a high level (Section 5) but without scripts or commands to replicate."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates across all benchmark tables (Tables 4, 6, 10, 12, etc.) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are reported despite numerous claims that PaLM 'outperforms' prior models. Comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports absolute performance numbers with baselines for context (e.g., Table 4 shows prior SOTA alongside PaLM scores, Table 10 shows GSM8K accuracy of 58% vs prior 55%). Percentage improvements and absolute differences are calculable from the comprehensive tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of evaluation examples used. For the multilingual NLG tasks, the paper mentions uniformly sampling 5,000 test examples (Section 6.6) but provides no justification for this choice."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.8 and Figure 17 show variance across checkpoints for a few tasks, revealing significant variation on WebQuestions. However, main results (Tables 4, 6, 10, 12) report single-run numbers with no variance or standard deviation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: GPT-3, GLaM, Gopher, Chinchilla, Megatron-Turing NLG, LaMDA, Codex, T5, and others across all evaluation sections. Table 4 compares against 6 prior large LMs."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Chinchilla (2022), Gopher (2021), Megatron-Turing NLG (2022), and GLaM (2021), which were contemporary state-of-the-art at the time. The paper also compares with Chinchilla in Section 13."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper trains 8B, 62B, and 540B models identically to study scale effects. Chain-of-thought vs direct prompting ablation is shown in Figure 10. Calculator vs no-calculator ablation for GSM8K (Table 10). Few-shot vs finetuning comparison (Table 8). Training longer ablation in Appendix F."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: EM and F1 for QA tasks, BLEU for translation, ROUGE-2/ROUGE-L/BLEURT-20 for generation (Section 6.6), pass@1 and pass@k for code tasks, accuracy for classification. Different tasks use appropriate metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "BIG-bench includes human performance baselines (both average and best) from crowdworkers (Section 6.2). The paper compares PaLM against these human baselines across 150 tasks. Winogender uses human performance baseline of 95.9% (Section 10.1.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on standard test splits. Section 6.1 uses the same splits as Du et al. (2021) and Brown et al. (2020). Table 9 reports SuperGLUE test set results. MMLU uses test set (Table 6). Section 8 analyzes data contamination."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-task breakdowns: Table 4 (29 individual NLP tasks), Table 6 (MMLU by category), Figure 5 (individual BIG-bench tasks), Figure 7 (24 BIG-bench Lite tasks), Table 12 (individual code tasks), Table 14 (translation by language pair)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 discusses tasks where PaLM fails (navigate, mathematical_induction). Figure 9 categorizes GSM8K errors. Figure 6 shows 35% of BIG-bench tasks where humans outperform PaLM. Section 10 discusses bias/toxicity failures. Table 43 lists tasks where humans significantly outperform PaLM."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: parallel layers showed quality degradation at 8B scale (Section 2); 1-shot CoQA is worse than prior SOTA (Table 4); training instability spikes occurred ~20 times (Section 5.1); toxicity increases with scale (Section 10.2); navigate and mathematical_induction show flat scaling (Figure 5c)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of SOTA on hundreds of benchmarks, outperforming human BIG-bench average, and breakthrough reasoning performance are all supported by Tables 4, Figure 3, and Section 6.3 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about scaling effects are supported by controlled comparisons: three model sizes trained identically on the same data (Section 2.1). Chain-of-thought ablation (Table 10) isolates prompting technique from model scale. The paper is appropriately cautious, noting confounds between factors (Section 13)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 13 explicitly discusses open questions about scaling trade-offs. Section 10.3 acknowledges fairness evaluations are limited to English. Section 11 discusses limitations of benchmark evaluations. The paper avoids overly broad claims, consistently qualifying results to specific benchmarks and settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 13 discusses alternative scaling strategies (training smaller models longer, as Chinchilla does). Section 8 addresses data contamination as an alternative explanation. Section 7 discusses memorization. Section 5.1 discusses training instability. The paper acknowledges training corpus quality as a confound."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports specific benchmark metrics (accuracy, F1, BLEU, pass@k) without overclaiming broader capabilities. Section 11 and Appendix E explicitly discuss that benchmark performance may not reflect real-world capabilities. Section 6.4 Discussion acknowledges the gap between pass@k and functional correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "PaLM model sizes (8B, 62B, 540B) are fully specified with architecture details in Table 1. For comparison models, specific versions are cited (GPT-3 175B, Codex 12B, Gopher 280B, etc.). The paper uses 'code-davinci-001' for Codex API (Section 6.4)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Chain-of-thought prompts are shown in Figures 1, 8, and Section 9. Few-shot exemplars for joke explanation and logical inference are provided. Code task prompts are shown in Figure 11. Translation and summarization prompts are described in Section 6.6 with actual text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 provides comprehensive hyperparameters: learning rate (10^-2 with decay), optimizer (Adafactor), momentum (β1=0.9), gradient clipping (1.0), batch size schedule, sequence length (2048), weight initialization, dropout settings, and finetuning hyperparameters (Section 6.1.2). Code inference uses nucleus sampling p=0.95, temperature 0.8 (Section 6.4)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. PaLM is evaluated via direct prompting (few-shot) and finetuning, not through an agentic pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data preprocessing: web page quality filtering with classifier, code deduplication via Levenshtein distance, programming language filtering by extension, license filtering for code. Table 2 gives mixing proportions. Appendix D provides a datasheet. SentencePiece tokenization is detailed in Section 2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.3 'Limitations' discusses limitations of bias analysis. Section 11 'Ethical Considerations' substantively discusses risks and limitations. Section 13 'Open Questions in Scaling' discusses what the study doesn't resolve."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: fairness analyses limited to English only (Section 10.3), bias benchmarks may not be portable to non-Western contexts (citing Sambasivan et al. 2021), benchmark contamination analyzed per-task (Section 8), checkpoint variance affecting results (Section 6.8, Figure 17)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.3 explicitly states fairness evaluations are limited to English. Section 13 states that conclusions about compute-optimal scaling cannot be drawn without ablation studies. Section 11 acknowledges benchmarks may not capture real-world performance. The paper explicitly does not claim to resolve optimal scaling (Section 13)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the training data nor raw evaluation outputs are released. Only aggregated benchmark results are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes training data collection: web pages filtered by quality classifier, books, Wikipedia, news, social media conversations, and GitHub code filtered by license and programming language. Table 2 gives proportions. Appendix D provides a detailed datasheet."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for this study. BIG-bench human baselines were collected by the BIG-bench collaboration, not by the PaLM authors. Data sources are standard public corpora/benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The training data pipeline is documented: source selection, quality filtering, deduplication, license filtering for code, language distribution (Table 29), and mixing proportions (Table 2). The datasheet in Appendix D provides additional pipeline details."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure. All authors are from Google Research, implying corporate funding, but no formal funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Google Research. Some are noted as Alphabet X or 'work done while at Google.' The affiliation is clearly stated on the first page."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Google has a direct commercial interest in demonstrating the capabilities of large language models trained on its infrastructure (TPU v4, Pathways). The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is provided. Google employees evaluating Google's infrastructure and models have potential financial interests that are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D states data was collected 2019-2021. Section 11 mentions 'spanning from very old texts to late 2021.' This establishes the training data timeframe."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 provides thorough data contamination analysis: categorizes 29 benchmarks into four contamination categories, reports clean vs full subset performance for 10 partially contaminated datasets (Table 18), and performs similar analysis for translation (Table 19)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 addresses contamination comprehensively: splits datasets into contaminated/clean subsets using 8-gram overlap, reports performance on clean subsets showing similar or better performance (Tables 18-19). BIG-bench canary string verified absent from training data (Section 6.2). Reasoning evaluation sets checked via n-gram overlap (Section 6.3.1)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted. BIG-bench human baselines were collected by the BIG-bench collaboration separately."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study was conducted by the PaLM authors."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency figures are reported for the evaluation runs despite the enormous model size (540B parameters)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports compute usage: PaLM 540B trained on 6144 TPU v4 chips for 1200 hours plus 3072 chips for 336 hours, totaling 2.56 × 10^24 FLOPs (29,600 PetaFLOP/s-days). Environmental impact: 271.43 tCO2e. Table 22 provides FLOP counts for all three model sizes."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 mentions bitwise determinism from checkpoints, but no multi-seed experiments are reported. Section 6.8 shows checkpoint variance (Figure 17) but this is variation across training checkpoints, not across random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be single-run. The paper does not state the number of runs for evaluation. Memorization analysis uses greedy decoding (Section 7). Toxicity analysis uses 25 continuations per prompt (Section 10.2), but main benchmark results are single-run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper mentions architecture choices (parallel layers, SwiGLU, multi-query attention) and some ablations, but does not report how many configurations were tried or the compute spent on search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Architecture choices are justified through ablation experiments (Section 2: parallel layers quality-neutral at 62B, multi-query attention quality-neutral). Finetuning checkpoints selected by validation performance (Section 6.6.1). Few-shot prompts follow prior work setups."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No multiple comparison correction is applied despite comparing across 29+ NLP benchmarks, 150 BIG-bench tasks, and multiple other evaluation suites. No statistical tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not acknowledge the bias of Google evaluating its own model. Comparisons against other models use published numbers from those models' papers, but no independent evaluation is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 13 and Figure 24 compare performance as a function of total training FLOP count across PaLM, Chinchilla, and Gopher. Table 21 provides FLOP counts alongside performance. The discussion explicitly addresses compute-efficiency trade-offs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 discusses that 'benchmark evaluations are often deficient in construct validity' (citing Raji et al. 2021). Section 6.4 Discussion acknowledges that pass@k based on small test suites can overestimate performance. The paper questions what benchmarks actually measure."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. PaLM is evaluated through direct prompting and finetuning, not through agentic scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly addresses temporal issues: training data collected 2019-2021, BIG-bench not available on Internet at data collection time. Section 6.2 verifies BIG-bench canary string absent from training data. Reasoning datasets verified clean via n-gram overlap."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the few-shot prompting setup or evaluation harnesses provide information that would not be available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 performs 8-gram overlap analysis between training data and evaluation sets, categorizing benchmarks into contamination types (wholesale, constructed from web, context on web, no significant overlap). Tables 18-19 compare clean vs full subset performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 uses 8-gram overlap analysis to detect contamination. Section 6.2 checks for BIG-bench canary string in training data. Section 6.3.1 verifies no n-gram overlap between training corpus and reasoning evaluation sets. Section 6.2 spot-checks model inputs/outputs for information leakage from gold labels."
       }
     }
   },
   "claims": [
     {
       "claim": "PaLM 540B achieves state-of-the-art few-shot results on 28 out of 29 widely evaluated English NLP benchmarks",
       "evidence": "Table 4 shows PaLM 540B outperforming prior SOTA on 28/29 tasks in few-shot setting, compared against GLaM, GPT-3, Megatron-Turing NLG, Gopher, Chinchilla, and LaMDA.",
       "supported": "strong"
     },
     {
       "claim": "PaLM 540B 5-shot outperforms average human performance on BIG-bench",
       "evidence": "Figure 3-left shows PaLM 540B 5-shot aggregate score exceeding average human performance on 58 common tasks. Figure 6 shows PaLM outperforms average humans on 65% of individual tasks.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-thought prompting with PaLM 540B outperforms finetuned SOTA on GSM8K",
       "evidence": "Table 10: PaLM 540B + chain-of-thought + calculator achieves 58% vs prior SOTA of 55% (GPT-3 + finetuning + chain-of-thought + calculator + verifier).",
       "supported": "strong"
     },
     {
       "claim": "Approximately 25% of BIG-bench tasks show discontinuous improvements from scale",
       "evidence": "Section 6.2 defines discontinuity metric and reports 25% of 150 tasks had discontinuity > +10% and 15% had > +20%. Examples shown in Figure 5b (english_proverbs, logical_sequence).",
       "supported": "strong"
     },
     {
       "claim": "PaLM achieves 46.2% model FLOPs utilization, significantly higher than prior work",
       "evidence": "Table 3 compares MFU: PaLM 46.2% vs GPT-3 21.3%, Gopher 32.5%, Megatron-Turing NLG 30.2%. Appendix B provides detailed calculation.",
       "supported": "strong"
     },
     {
       "claim": "Scaling improvements have not plateaued for large language models",
       "evidence": "Figure 3 shows log-linear scaling on BIG-bench. Table 4 shows consistent improvements from 62B to 540B. Section 13 compares against Chinchilla showing continued improvements with compute.",
       "supported": "moderate"
     },
     {
       "claim": "PaLM outperforms supervised translation baselines on German-English and Romanian-English",
       "evidence": "Table 14 shows PaLM 540B few-shot BLEU of 47.5 (de-en) vs finetuned SOTA 41.2, and 43.8 (ro-en) vs 39.1. However, these supervised baselines may be outdated as noted by the authors.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All authors are Google employees evaluating a Google model trained on Google infrastructure (TPU v4, Pathways). Google has direct commercial interest in demonstrating scaling benefits and infrastructure capabilities. No independent evaluation is included."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "Despite hundreds of comparisons across dozens of benchmarks, no statistical significance tests, confidence intervals, or error bars are reported for the main results. Claims of 'outperforming' are based on comparing point estimates."
     },
     {
       "flag": "Single-run results",
       "detail": "Main benchmark results appear to be single-run evaluations. Section 6.8 shows significant variance across checkpoints for some tasks (WebQuestions in Figure 17), suggesting reported results may be sensitive to evaluation timing."
     },
     {
       "flag": "Potentially outdated translation baselines",
       "detail": "The paper acknowledges that supervised translation baselines 'might be outdated' (Section 6.5) when claiming PaLM outperforms supervised models on some language pairs."
     }
   ],
   "cited_papers": [
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing the few-shot evaluation paradigm for large language models that PaLM extends."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Establishes power-law scaling relationships for LMs that PaLM's discontinuous improvements challenge."
     },
     {
       "title": "Training Compute-Optimal Large Language Models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla paper arguing for training smaller models longer, directly compared against PaLM in Section 13."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "arxiv_id": "2201.11903",
       "relevance": "Chain-of-thought prompting technique that PaLM combines with scale to achieve breakthrough reasoning results."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval paper establishing code generation benchmarks that PaLM is evaluated on."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "MBPP benchmark and code synthesis evaluation methodology used in PaLM's code evaluation."
     },
     {
       "title": "Scaling Language Models: Methods, Analysis & Insights from Training Gopher",
       "authors": ["Jack W. Rae", "Sebastian Borgeaud", "Trevor Cai"],
       "year": 2021,
       "arxiv_id": "2112.11446",
       "relevance": "Gopher paper providing baselines and evaluation methodology that PaLM extends."
     },
     {
       "title": "GLaM: Efficient Scaling of Language Models with Mixture-of-Experts",
       "authors": ["Nan Du", "Yanping Huang", "Andrew M. Dai"],
       "year": 2021,
       "arxiv_id": "2112.06905",
       "relevance": "GLaM sparse model providing baselines for PaLM evaluation; PaLM's training dataset is based on GLaM's."
     },
     {
       "title": "Quantifying Memorization Across Neural Language Models",
       "authors": ["Nicholas Carlini", "Daphne Ippolito", "Matthew Jagielski"],
       "year": 2022,
       "arxiv_id": "2202.07646",
       "relevance": "Memorization analysis methodology applied to PaLM in Section 7."
     },
     {
       "title": "Finetuned Language Models Are Zero-Shot Learners",
       "authors": ["Jason Wei", "Maarten Bosma", "Vincent Y. Zhao"],
       "year": 2022,
       "relevance": "FLAN instruction tuning approach compared against PaLM's few-shot capabilities."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2021,
       "relevance": "MMLU benchmark used to evaluate PaLM's multitask knowledge capabilities."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems",
       "authors": ["Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "GSM8K benchmark and verifier approach that PaLM's chain-of-thought prompting outperforms."
     }
   ]
 }