ai-research-survey

scan-v5.json (26203B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "LLaMA: Open and Efficient Foundation Language Models",
     "authors": [
       "Hugo Touvron",
       "Thibaut Lavril",
       "Gautier Izacard",
       "Xavier Martinet",
       "Marie-Anne Lachaux",
       "Timothee Lacroix",
       "Baptiste Rozière",
       "Naman Goyal",
       "Eric Hambro",
       "Faisal Azhar",
       "Aurelien Rodriguez",
       "Armand Joulin",
       "Edouard Grave",
       "Guillaume Lample"
     ],
     "year": 2023,
     "venue": "arXiv.org",
     "arxiv_id": "2302.13971",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are substantiated: LLaMA-13B outperforming GPT-3 on most benchmarks is shown in Tables 3–9, and LLaMA-65B's competitiveness with Chinchilla-70B and PaLM-540B is demonstrated across common sense reasoning, QA, and MMLU.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper adopts architectural changes (RMSNorm, SwiGLU, RoPE) and attributes improved training stability and performance to them, but provides no ablation studies to isolate the contribution of each change.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'state-of-the-art models' can be trained on public data, but LLaMA-65B underperforms Chinchilla-70B and PaLM-540B on MMLU (63.4 vs 67.5/69.3), which is not acknowledged in the abstract's framing.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for LLaMA's performance gains, such as whether better data curation alone (rather than more training tokens) explains results; MMLU underperformance is attributed to fewer books/academic papers but no competing hypotheses are considered.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently frames claims in terms of benchmark accuracy scores and does not conflate benchmark performance with broader notions of general intelligence or real-world capability.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. Section 5 covers bias and toxicity but this is an evaluation of model behavior rather than a methodological limitations discussion.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes that TriviaQA uses a different split than GPT-3/PaLM and that SIQA benchmark may be unreliable due to high variance, but there is no systematic discussion of threats to the validity of comparative claims.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions instruction finetuning is 'beyond the scope of this paper' and future larger models are planned, but does not explicitly state what the presented results do NOT show or where the findings should not be applied.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure statement is present in the paper; the work is identified as from Meta AI by affiliation only.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Meta AI, which is clearly stated on the paper's first page.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta AI employees are evaluating and promoting models they themselves built; the funder and developers are identical, creating an inherent conflict of interest.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests declaration anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": false,
         "justification": "Core terms like 'foundation language models', 'state-of-the-art', and 'competitive' are used throughout without formal definition; the boundary for what counts as competitive performance is not made explicit.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The contribution is explicitly stated: training competitive LLMs using only publicly available data and releasing them to the research community to democratize access to large language models.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 provides substantive engagement with prior LLM work covering architecture history, scaling laws, and concurrent open models; throughout the paper results are directly positioned against GPT-3, Chinchilla, PaLM, OPT, and BLOOM.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Source code is released at https://github.com/facebookresearch/llama, referenced in the paper's first footnote.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Training data consists entirely of publicly available datasets (CommonCrawl, C4, GitHub, Wikipedia, Gutenberg, Books3, ArXiv, StackExchange); all evaluation benchmarks are standard public benchmarks.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes hardware (2048 A100-80GB GPUs) and references the xformers library but provides no requirements file, Dockerfile, or equivalent environment specification.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions are provided in the paper; reproduction would require inferring details from the code repository.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "No confidence intervals or error bars are reported for any benchmark results in Tables 3–14; all results are single point estimates.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are performed for any comparative claims; performance differences are reported as raw accuracy values without significance testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute performance values and relative comparisons (e.g., '13B outperforms GPT-3 on most benchmarks despite being 10x smaller', specific accuracy gaps) are reported throughout the tables.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The paper relies on established benchmarks without discussing sample sizes, statistical power, or whether the benchmarks are large enough to support the comparative claims made.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or run-to-run variability is reported for any result; all figures are single-run point estimates.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive baselines included: GPT-3 (175B), Gopher (280B), Chinchilla (70B), PaLM (8B, 62B, 540B), OPT, GPT-J, and GPT-NeoX across all evaluation sections.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines (Chinchilla, PaLM, GPT-3, Gopher) are the leading contemporary models from 2021–2022, directly competitive at the time of this work.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "No ablation studies are conducted; the paper adopts multiple architectural changes (RMSNorm, SwiGLU, RoPE) from prior work without ablating their individual contributions.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation spans 20 benchmarks across eight domains: common sense reasoning (8 tasks), closed-book QA, reading comprehension, math reasoning (MATH, GSM8k), code generation (HumanEval, MBPP), MMLU, and safety benchmarks.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "No human evaluation of model outputs is conducted; all evaluation uses automated benchmark scoring.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All evaluation uses held-out test/dev sets of established benchmarks (NaturalQuestions test split, TriviaQA filtered dev set, MMLU test sets, etc.).",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "MMLU results are broken down by domain (Humanities, STEM, Social Sciences, Other) in Table 9 and per-task in Table 16; code results are broken down by benchmark and metric (pass@1, pass@100).",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper identifies where LLaMA underperforms: behind Chinchilla/PaLM on MMLU (attributed to fewer books in training), lower BoolQ scores, WinoGrande variance issues, and high hallucination rates on TruthfulQA.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results are explicitly reported: LLaMA-65B lags Chinchilla-70B and PaLM-540B on MMLU by several points, TruthfulQA correctness rates are low (0.57 max), and SIQA is flagged as potentially unreliable.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 provides exact specifications for all four model sizes: parameter count, hidden dimension, number of heads, layers, learning rate, batch size, and training token count.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A shows formatted evaluation examples for NaturalQuestions and TriviaQA with the prepended instruction string; evaluation protocol details are provided for each benchmark.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.3 and Table 2 report optimizer (AdamW), β1=0.9, β2=0.95, weight decay=0.1, gradient clipping=1.0, cosine LR schedule, warmup steps=2000, and per-model learning rates and batch sizes.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This is a base LLM pretraining paper with no agentic scaffolding or tool use.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 provides detailed preprocessing documentation for each data source, including deduplication methods, quality filtering heuristics, language identification, tokenizer choice (BPE via SentencePiece), and epoch counts per source.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The preprocessed training data mixture is not released; only the constituent public sources are identified. The specific preprocessed dataset used to train LLaMA cannot be independently retrieved.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 2.1 describes data collection and processing for all seven data sources with specific pipelines (CCNet, C4 preprocessing, GitHub license filtering, Wikipedia dumps, BPE tokenization).",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants are recruited; this is a pretraining study using web-crawled and curated text data.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full data pipeline from raw source to tokenized training data is documented, including sampling proportions (Table 1), epoch counts, disk sizes, and preprocessing steps for each source.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies Wikipedia dumps from June–August 2022 and CommonCrawl dumps from 2017–2020, but no single explicit training data cutoff date is stated for the combined corpus.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No analysis of potential overlap between training data (including 2022 CommonCrawl, ArXiv, StackExchange) and evaluation benchmarks is performed or discussed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Many evaluation benchmarks (BoolQ, HellaSwag, MMLU, NaturalQuestions, TriviaQA) were publicly available well before the training data collection window; no decontamination or overlap analysis is reported.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper qualitatively notes that LLaMA-13B runs on a single V100 and smaller models run on a single GPU, but provides no quantitative latency or inference cost measurements.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 and Table 15 explicitly state training compute: 2048 A100-80GB GPUs for approximately 5 months, with per-model GPU-hours (e.g., 1,022,362 GPU-hours for 65B), total power (449 MWh), and carbon emissions (173 tCO2eq).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "LLaMA-13B outperforms GPT-3 (175B) on most benchmarks despite being more than 10x smaller in parameter count.",
       "evidence": "Tables 3–9 show LLaMA-13B exceeding GPT-3 on BoolQ, PIQA, HellaSwag, WinoGrande, ARC, NaturalQuestions, TriviaQA, RACE, and MMLU (46.9 vs 43.9).",
       "supported": "strong"
     },
     {
       "claim": "LLaMA-65B is competitive with Chinchilla-70B and PaLM-540B.",
       "evidence": "LLaMA-65B outperforms Chinchilla-70B on most common sense reasoning benchmarks (Table 3) and matches PaLM-540B on several tasks, though it lags on MMLU (63.4 vs 67.5/69.3).",
       "supported": "moderate"
     },
     {
       "claim": "State-of-the-art language model performance can be achieved using only publicly available data.",
       "evidence": "All training data sources (CommonCrawl, C4, GitHub, Wikipedia, etc.) are publicly available, and the resulting models match or exceed proprietary-data models on most benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "Training smaller models on more tokens yields better inference efficiency than training larger models to compute-optimality.",
       "evidence": "The 7B model continues to improve after 1T tokens (beyond Chinchilla's recommended stopping point), and the 13B model achieves GPT-3 quality at far lower inference cost.",
       "supported": "moderate"
     },
     {
       "claim": "Toxicity in generated text increases with model size within the LLaMA family.",
       "evidence": "Table 11 shows RealToxicityPrompts scores increasing from 7B (0.106) to 65B (0.128) on basic prompts and more sharply on respectful prompts (0.081 to 0.141).",
       "supported": "moderate"
     },
     {
       "claim": "LLaMA-65B exhibits gender bias in coreference resolution, performing worse on 'gotcha' cases where pronouns don't match occupational stereotypes.",
       "evidence": "Table 13 shows WinoGender gotcha accuracy drops to 75.0% (her/her/she) and 63.3% (his/him/he) versus overall 77.5%, demonstrating the model relies on occupational gender stereotypes.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "LLaMA demonstrates that competitive foundation language models can be trained using only publicly available data, with LLaMA-13B outperforming GPT-3 (175B) across most benchmarks at 10x fewer parameters. The work extends Chinchilla scaling insights to the inference-budget regime, showing continued improvement from training smaller models on more tokens. Safety evaluations reveal persistent bias and toxicity issues, with toxicity increasing with scale and significant gender bias in coreference resolution tasks. The compute and carbon footprint are transparently documented, and model weights are released.",
   "red_flags": [
     {
       "flag": "No ablation studies",
       "detail": "Three architectural modifications (RMSNorm, SwiGLU, RoPE) are adopted together without any ablation to isolate their individual contributions to performance or training stability."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims across 20+ benchmarks are presented as raw accuracy numbers without confidence intervals, significance tests, or error bars, making it impossible to assess which differences are meaningful."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "Training data includes 2017–2022 CommonCrawl, ArXiv, and StackExchange data that almost certainly contains benchmark examples (MMLU, HellaSwag, BoolQ, etc.); no decontamination analysis is performed."
     },
     {
       "flag": "Self-evaluation by developer",
       "detail": "Meta AI employees evaluate their own models with no independent third-party verification; methodology choices (benchmark selection, evaluation protocols) may favor the presented models."
     },
     {
       "flag": "Inconsistent evaluation splits",
       "detail": "TriviaQA uses the filtered dev set rather than the unfiltered test set used by GPT-3 and PaLM (acknowledged in Appendix A), making direct comparison potentially misleading."
     },
     {
       "flag": "Overclaiming on MMLU",
       "detail": "The abstract claims LLaMA-65B is 'competitive with the best models,' but it underperforms Chinchilla-70B (63.4 vs 67.5) and PaLM-540B (63.4 vs 69.3) on MMLU without this caveat in the abstract."
     }
   ],
   "cited_papers": [
     {
       "title": "Training Compute-Optimal Large Language Models (Chinchilla)",
       "relevance": "Central to LLaMA's motivation: the paper extends Chinchilla's compute-optimal scaling laws to the inference-budget regime, training smaller models on more tokens."
     },
     {
       "title": "Language Models are Few-Shot Learners (GPT-3)",
       "relevance": "Primary baseline throughout; LLaMA-13B is benchmarked against GPT-3-175B as the key competitive comparison."
     },
     {
       "title": "PaLM: Scaling Language Modeling with Pathways",
       "relevance": "Key architectural inspiration (SwiGLU) and major comparison baseline across all evaluation tasks."
     },
     {
       "title": "OPT: Open Pre-trained Transformer Language Models",
       "relevance": "Most relevant prior open-source LLM comparison; LLaMA is positioned as an improvement over OPT in both performance and openness."
     },
     {
       "title": "Scaling Laws for Neural Language Models (Kaplan et al.)",
       "relevance": "Foundational scaling law work that LLaMA builds upon and extends."
     },
     {
       "title": "Emergent Abilities of Large Language Models (Wei et al.)",
       "relevance": "Contextualizes why scaling matters for few-shot task performance."
     },
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Primary benchmark for knowledge evaluation; results expose LLaMA's data composition limitations."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code (Codex/HumanEval)",
       "relevance": "Provides HumanEval benchmark and pass@k methodology used for code generation evaluation."
     },
     {
       "title": "BLOOM: A 176B-Parameter Open-Access Multilingual Language Model",
       "relevance": "Contemporary open LLM comparison for carbon footprint analysis and open-source LLM landscape context."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly usable open model weights released publicly; 7B/13B models run on consumer hardware, enabling wide practitioner adoption."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that proprietary data is necessary for competitive LLM performance; the inference-budget framing of scaling laws is a meaningful reframing."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Documents bias and toxicity but frames these as known issues; primary concern is open release enabling misuse, which is not deeply addressed."
     },
     "drama_conflict": {
       "score": 2,
       "justification": "Directly challenges closed-source LLM providers (OpenAI, Google) by releasing competitive open weights; fueled significant open vs. closed AI debate in 2023."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Openly released weights at GitHub; any researcher can download and run these models immediately on accessible hardware."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Meta AI (Facebook Research), highly cited paper that spawned the entire LLaMA/Llama 2/3 ecosystem and directly influenced Alpaca, Vicuna, and dozens of derivative models."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "39856808",
         "title": "LLaMA: Open and Efficient Foundation Language Models (2023)",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39856808",
         "created_at": "2024-03-28T20:19:31Z"
       }
     ],
     "top_points": 2,
     "total_points": 2,
     "total_comments": 0
   }
 }