ai-research-survey

scan.json (31996B)

---

 {
   "paper": {
     "title": "LLaMA: Open and Efficient Foundation Language Models",
     "authors": [
       "Hugo Touvron",
       "Thibaut Lavril",
       "Gautier Izacard",
       "Xavier Martinet",
       "Marie-Anne Lachaux",
       "Timothee Lacroix",
       "Baptiste Rozière",
       "Naman Goyal",
       "Eric Hambro",
       "Faisal Azhar",
       "Aurelien Rodriguez",
       "Armand Joulin",
       "Edouard Grave",
       "Guillaume Lample"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2302.13971"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLaMA is a family of foundation language models (7B–65B parameters) trained exclusively on publicly available data, demonstrating that state-of-the-art performance does not require proprietary datasets. LLaMA-13B outperforms GPT-3 (175B) on most benchmarks despite being 10× smaller, and LLaMA-65B is competitive with Chinchilla-70B and PaLM-540B. The paper challenges the Chinchilla scaling assumption by showing that smaller models trained on more tokens can match or exceed larger models, shifting the optimization target from training compute to inference efficiency. Responsible AI evaluations reveal increasing toxicity with model scale and societal biases captured from web training data.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link: https://github.com/facebookresearch/llama (footnote 1, Section 1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All training data sources are publicly available and explicitly identified in Section 2.1 and Table 1: CommonCrawl (via CCNet), C4, public GitHub, Wikipedia, Gutenberg, Books3, ArXiv, and StackExchange. The paper emphasizes 'only using data that is publicly available, and compatible with open sourcing.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware (2048 A100-80GB GPUs) and mentions the xformers library and SentencePiece tokenizer, but does not provide a requirements.txt, Dockerfile, or detailed software environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper describes the training procedure and hyperparameters in detail, there are no step-by-step reproduction instructions, scripts, or a 'Reproducing Results' section. The GitHub repository is linked but the paper itself lacks specific instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3–10 are reported as single point estimates with no confidence intervals, error bars, or uncertainty notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Claims of 'outperforms' and 'competitive with' are based solely on comparing raw numbers across models (e.g., 'LLaMA-65B outperforms Chinchilla-70B on all reported benchmarks but BoolQ')."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The comparison tables (Tables 3–10) provide absolute scores for all baseline models alongside LLaMA results, providing full baseline context for assessing magnitude of improvements (e.g., BoolQ: LLaMA-65B 85.3 vs Chinchilla 83.7 vs GPT-3 60.5). The parameter efficiency comparison (13B vs 175B) also provides practical effect size context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of evaluation examples used. Standard benchmark sizes are used without discussion of whether they provide sufficient statistical power for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or multi-run results are reported. All benchmark results appear to be from single runs. The only exception is pass@k for code generation, which uses sampling-based estimates but does not report variance across independent runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: GPT-3 (175B), Gopher (280B), Chinchilla (70B), PaLM (8B/62B/540B), OPT, GPT-J, GPT-Neo, LaMDA, and Minerva. Numbers are taken from the corresponding papers (Section 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Chinchilla (2022), PaLM (2022), and OPT (2022), which were the most recent and competitive models at time of publication. Older models (GPT-3, 2020) are also included for completeness."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The paper adopts architectural components from prior work (pre-normalization from GPT-3, SwiGLU from PaLM, RoPE from GPT-Neo) but does not ablate them to measure individual contributions. The scaling analysis (4 model sizes) is not an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates across 20+ benchmarks spanning commonsense reasoning (8 tasks), question answering (exact match), reading comprehension, mathematical reasoning (MATH, GSM8k), code generation (pass@1, pass@k), MMLU, toxicity (RealToxicityPrompts), bias (CrowS-Pairs, WinoGender), and truthfulness (TruthfulQA)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluations are automated: benchmark accuracy, exact match, pass@k, PerspectiveAPI for toxicity, and perplexity-based bias measurement. No human evaluation of model outputs is conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard benchmark test splits are used. Appendix A specifies: Natural Questions uses 'the test split used for open-domain question answering containing 3610 questions.' TriviaQA uses the dev set (test server unavailable). MMLU uses 'test sets' (Table 16)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive breakdowns are provided: per-benchmark results across all model sizes (Tables 3–10), per-domain MMLU results (Table 16 in appendix, 57 tasks), per-category bias results (Table 12, 9 categories), and per-pronoun WinoGender results (Table 13)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where LLaMA underperforms: MMLU gap vs Chinchilla/PaLM (Section 3.6, attributed to limited books/academic data), increasing toxicity with model size (Table 11), gender bias in co-reference resolution (Section 5.3, gotcha examples), and hallucination tendency (TruthfulQA, Section 5.4)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "LLaMA-65B falls behind Chinchilla-70B and PaLM-540B on MMLU (Section 3.6). Toxicity increases with model size (Table 11). Gender bias is demonstrated via WinoGender gotcha cases (Table 13). SIQA shows high variance suggesting benchmark unreliability (Section 3.7). TruthfulQA scores are acknowledged as low (Section 5.4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: (1) 'train on trillions of tokens' — confirmed by Table 1 (1.0T–1.4T tokens). (2) 'LLaMA-13B outperforms GPT-3 on most benchmarks' — confirmed by Tables 3–8. (3) 'LLaMA-65B is competitive with Chinchilla-70B and PaLM-540B' — confirmed across benchmarks. (4) 'using publicly available datasets exclusively' — confirmed by Section 2.1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claim is that training smaller models on more tokens (beyond Chinchilla-optimal) yields competitive performance. This is supported by systematic variation of model sizes (7B–65B) with controlled training procedures, training loss curves (Figure 1), and benchmark performance evolution during training (Figure 2). The study design (same architecture, same data, different scales) provides adequate evidence for this existence proof."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to tested settings. The abstract specifies exact model sizes and comparison targets. Claims are made about specific benchmarks with specific model sizes. The paper frames its contribution as an existence proof ('it is possible to train state-of-the-art models using publicly available datasets') rather than claiming universal superiority."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Several alternative explanations are discussed: MMLU underperformance attributed to limited books/academic data in training (Section 3.6). Toxicity-size relationship 'may only apply within a model family' based on Chinchilla vs Gopher comparison (Section 5.1). PerspectiveAPI methodology differences affecting cross-paper comparisons (Section 5.1). SIQA variance suggesting benchmark unreliability (Section 3.7)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It claims competitive benchmark performance and demonstrates it on specific benchmarks. It does not frame benchmark accuracy as a broader proxy for 'intelligence' or 'understanding.' The inference efficiency argument is clearly framed in terms of specific model sizes and benchmark scores."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model architectures are specified in Table 2 (parameters, dimensions, heads, layers, learning rates, batch sizes, token counts for all 4 model sizes). Baseline models are identified by exact sizes (GPT-3 175B, Chinchilla 70B, PaLM 8B/62B/540B, etc.) with results from their corresponding papers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A provides formatted prompt examples for Natural Questions and TriviaQA (Figure 3), including the prefix string 'Answer these questions:\\n'. The paper references the lm-evaluation-harness framework (Gao et al., 2021) for other benchmarks. Evaluation methodology is described, including likelihood normalization approaches from Brown et al. (2020)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 reports optimizer hyperparameters: AdamW with β1=0.9, β2=0.95, weight decay 0.1, gradient clipping 1.0, 2000 warmup steps, cosine learning rate schedule. Table 2 provides per-model learning rates and batch sizes. Code generation evaluation specifies temperature 0.1 for pass@1 and 0.8 for pass@100."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are evaluated directly on benchmarks without tool use, retrieval, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 provides detailed preprocessing documentation for each data source: CCNet pipeline for CommonCrawl (deduplication, language identification, quality filtering), C4 preprocessing (heuristic filtering), GitHub (license filtering, line length heuristics, boilerplate removal, file-level deduplication), Wikipedia (hyperlink/comment removal), Books (90% overlap deduplication), ArXiv (section/bibliography removal, macro expansion), StackExchange (HTML tag removal, score sorting). BPE tokenization with SentencePiece is described."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('Bias, Toxicity and Misinformation') is a dedicated multi-subsection evaluation of model limitations, covering RealToxicityPrompts (5.1), CrowS-Pairs bias (5.2), WinoGender bias (5.3), and TruthfulQA (5.4). While not titled 'Limitations,' it serves as a substantive limitations evaluation with multiple subsections and quantitative analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'We do not have control over the pipeline used by the third-party PerspectiveAPI, making comparison with previous models difficult' (Section 5.1). MMLU underperformance attributed to specific data composition ('limited amount of books and academic papers,' Section 3.6). SIQA benchmark reliability questioned due to high variance (Section 3.7). Bias evaluations 'are not sufficient to fully understand the risks' (Section 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it acknowledges specific weaknesses (MMLU, toxicity), it does not draw explicit boundaries around the scope of its performance claims or state what settings/populations/tasks are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The processed training data is not released. While the source datasets are publicly available (CommonCrawl, C4, etc.), the specific processed and filtered versions used for training are not provided. Model outputs on benchmarks are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 describes data collection in detail for each source: CommonCrawl dumps from 2017–2020, C4 from Raffel et al. 2020, GitHub from Google BigQuery (Apache/BSD/MIT licensed), Wikipedia June–August 2022 (20 languages), Gutenberg and Books3, ArXiv LaTeX files, and StackExchange (28 largest websites). Sampling proportions and disk sizes are provided in Table 1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public datasets and benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full data pipeline is documented in Section 2.1: raw data sources → preprocessing (deduplication, language identification, quality filtering per source) → BPE tokenization with SentencePiece → training data (~1.4T tokens). Each source has its filtering steps described. Table 1 shows final sampling proportions and epochs."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding statement is provided. The Acknowledgments section thanks individual Meta AI colleagues but does not disclose funding sources or grants."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated as 'Meta AI' in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta AI is both the employer/funder and the developer of the models being evaluated. Meta has a commercial interest in demonstrating that its models are competitive with proprietary alternatives (GPT-3, PaLM, Chinchilla)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data time periods are partially stated: CommonCrawl 2017–2020, Wikipedia June–August 2022. However, GitHub, ArXiv, StackExchange, and Books sources lack specific time period information. No single training data cutoff date is explicitly stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples appeared in the training data. Many evaluation benchmarks (BoolQ 2019, PIQA 2020, HellaSwag 2019, ARC 2018, etc.) predate the CommonCrawl collection and could have been included."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Benchmark contamination is not addressed despite significant risk. Many benchmarks (PIQA, BoolQ, HellaSwag, WinoGrande, ARC, OBQA, NaturalQuestions, TriviaQA, HumanEval) were published before the training data collection period and their contents are available on the web. No decontamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Inference cost is not systematically reported. The paper mentions that LLaMA-13B 'can be run on a single V100 GPU during inference' (Section 3.2) and that training processes '380 tokens/sec/GPU' (Section 2.4), but no inference latency, throughput, or cost-per-query numbers are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 and Table 15 provide detailed compute budgets: GPU-hours for each model (7B: 82,432; 13B: 135,168; 33B: 530,432; 65B: 1,022,362), total power consumption (36–449 MWh), and carbon emissions (14–173 tCO2eq). Hardware specified as A100-80GB GPUs. Total development estimated at 2,638 MWh over ~5 months on 2048 GPUs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single training runs for each model size."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Given the massive compute cost, results are likely from single runs, but this is not explicitly acknowledged."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Architecture and hyperparameters are adopted from prior work (GPT-3, PaLM, GPT-Neo), but the paper does not state whether any configuration search was performed or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Architecture choices are justified by citing prior successful models: pre-normalization from GPT-3, SwiGLU from PaLM, RoPE from GPT-Neo (Section 2.2). All four model sizes are reported rather than cherry-picking the best. The approach follows Chinchilla scaling laws for training duration."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction is applied. The paper makes comparative claims across 20+ benchmarks and 4+ baseline models without any statistical testing or correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge the bias of evaluating their own system. While they use baseline numbers from other papers (reducing re-implementation bias), they do not discuss the potential for favorable evaluation choices or setup advantages."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is a core contribution. The paper explicitly argues that compute-performance tradeoffs favor smaller models trained longer (challenging Chinchilla scaling laws). Figure 1 shows training loss vs tokens. Table 15 provides compute budgets. Performance is compared across model sizes (7B–65B) against larger models (GPT-3 175B, PaLM 540B), directly addressing the compute-performance relationship."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether benchmarks actually measure what is claimed, with one minor exception: SIQA is noted as showing 'a lot of variance in performance, that may indicate that this benchmark is not reliable' (Section 3.7). No systematic discussion of construct validity for the other 19+ benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly on benchmarks without tools, agents, or multi-step workflows."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Temporal leakage is not discussed. Many benchmarks (HellaSwag 2019, PIQA 2020, BoolQ 2019, etc.) were created before the CommonCrawl dumps (2017–2020), meaning benchmark content could appear in training data. No temporal analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Feature leakage is not discussed. No analysis of whether evaluation setup provides information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Non-independence of training and test data is not addressed. CommonCrawl data could contain web pages that include benchmark questions and answers, creating train-test overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipeline is described."
       }
     }
   },
   "claims": [
     {
       "claim": "LLaMA-13B outperforms GPT-3 (175B) on most benchmarks despite being 10× smaller",
       "evidence": "Tables 3–8 show LLaMA-13B exceeding GPT-3 on commonsense reasoning (7/8 tasks), NaturalQuestions (20.1 vs 14.6 zero-shot), TriviaQA (56.6 vs 43.5 zero-shot), RACE, MATH/GSM8k, code generation (HumanEval 15.8 vs 14.0), and MMLU (46.9 vs 43.9).",
       "supported": "strong"
     },
     {
       "claim": "LLaMA-65B is competitive with Chinchilla-70B and PaLM-540B",
       "evidence": "Tables 3–9 show LLaMA-65B outperforming Chinchilla on all commonsense benchmarks except BoolQ, surpassing PaLM-540B on most except BoolQ and WinoGrande. On MMLU (Table 9), LLaMA-65B (63.4%) trails Chinchilla (67.5%) and PaLM-540B (69.3%).",
       "supported": "strong"
     },
     {
       "claim": "State-of-the-art performance can be achieved using publicly available datasets exclusively",
       "evidence": "Section 2.1 describes all data sources (CommonCrawl, C4, public GitHub, Wikipedia, Gutenberg, Books3, ArXiv, StackExchange) — all publicly available. Model performance is competitive with Chinchilla and PaLM which use proprietary data.",
       "supported": "strong"
     },
     {
       "claim": "Performance continues to improve when training beyond Chinchilla-optimal token counts",
       "evidence": "Figure 1 shows training loss continuing to decrease for all model sizes up to 1.0T–1.4T tokens. Figure 2 shows benchmark performance continuing to improve during training. The 7B model trained on 1T tokens (5× the Chinchilla-recommended 200B) still improves.",
       "supported": "strong"
     },
     {
       "claim": "Brief instruction finetuning improves MMLU performance",
       "evidence": "Table 10 shows LLaMA-I (65B) achieving 68.9% on MMLU vs LLaMA-65B at 63.4%, a 5.5-point improvement from instruction finetuning following Chung et al. (2022) protocol.",
       "supported": "moderate"
     },
     {
       "claim": "Toxicity increases with model size",
       "evidence": "Table 11 shows toxicity scores increasing from 0.106 (7B) to 0.128 (65B) for basic prompts and from 0.081 (7B) to 0.141 (65B) for respectful prompts. However, the authors note PerspectiveAPI methodology differences limit comparisons.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are Meta AI employees evaluating Meta's own LLaMA models. While baselines use numbers from other papers (reducing some bias), the evaluation setup, benchmark selection, and presentation choices are controlled by the model developers."
     },
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All results across 20+ benchmarks are reported as single point estimates with no confidence intervals, standard deviations, or multi-run statistics. Given the massive compute cost, single-run results are understandable but claims of 'outperforming' specific models by small margins (e.g., 1–3 points) are not statistically validated."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The models are trained on web data (CommonCrawl 2017–2020) that could contain benchmark questions and answers. Many benchmarks (HellaSwag, PIQA, BoolQ, ARC, WinoGrande, NaturalQuestions, TriviaQA) predate the training data. No decontamination analysis is performed despite this significant risk."
     },
     {
       "flag": "No ablation of architectural choices",
       "detail": "The paper adopts three architectural changes from prior work (pre-normalization, SwiGLU, RoPE) without ablating their individual contributions. It is unclear how much each modification contributes to the final performance."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper; foundational work on large language model few-shot capabilities and primary baseline for LLaMA comparisons."
     },
     {
       "title": "Palm: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "relevance": "PaLM scaling paper; key baseline for LLaMA and source of architectural choices (SwiGLU activation)."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "relevance": "Chinchilla scaling laws paper; LLaMA directly challenges its recommendations by training smaller models longer."
     },
     {
       "title": "OPT: Open pre-trained transformer language models",
       "authors": ["Susan Zhang", "Stephen Roller", "Naman Goyal"],
       "year": 2022,
       "arxiv_id": "2205.01068",
       "relevance": "Open-source LLM release from Meta; key comparison point for open model availability and carbon footprint."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Codex/HumanEval paper; introduces the code generation benchmark used to evaluate LLaMA and establishes pass@k methodology."
     },
     {
       "title": "BLOOM: A 176b-parameter open-access multilingual language model",
       "authors": ["Teven Le Scao", "Angela Fan", "Christopher Akiki"],
       "year": 2022,
       "arxiv_id": "2211.05100",
       "relevance": "Open-access multilingual LLM; comparison point for open model release and carbon footprint analysis."
     },
     {
       "title": "Measuring massive multitask language understanding",
       "authors": ["Dan Hendrycks", "Collin Burns", "Steven Basart"],
       "year": 2020,
       "relevance": "MMLU benchmark paper; widely-used LLM evaluation benchmark where LLaMA shows relative weakness."
     },
     {
       "title": "Emergent abilities of large language models",
       "authors": ["Jason Wei", "Yi Tay", "Rishi Bommasani"],
       "year": 2022,
       "arxiv_id": "2206.07682",
       "relevance": "Studies the effect of scaling on LLM abilities; directly relevant to LLaMA's scaling analysis and claims about model size vs capability."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Establishes neural scaling laws that LLaMA builds upon, showing how model and dataset size relate to performance."
     },
     {
       "title": "Realtoxicityprompts: Evaluating neural toxic degeneration in language models",
       "authors": ["Samuel Gehman", "Suchin Gururangan", "Maarten Sap"],
       "year": 2020,
       "arxiv_id": "2009.11462",
       "relevance": "Toxicity evaluation benchmark used to assess LLaMA's potential for generating toxic content."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "MBPP code generation benchmark paper; used to evaluate LLaMA's code generation capabilities."
     }
   ]
 }