ai-research-survey

scan.json (31278B)

---

 {
   "paper": {
     "title": "The Llama 3 Herd of Models",
     "authors": ["Llama Team", "AI @ Meta"],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2407.21783"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Llama 3 405B performs comparably to GPT-4 across a wide range of tasks including general knowledge, coding, math, and multilingual benchmarks. The paper demonstrates scaling laws that accurately predict downstream performance across four orders of magnitude of compute. Human evaluations show Llama 3 405B is competitive with GPT-4o and Claude 3.5 Sonnet. The paper also presents preliminary multimodal (vision, video, speech) extensions that are competitive but not yet released.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states models are publicly released under the Llama 3 Community License at https://llama.meta.com. FP8 quantization kernels are released at https://github.com/pytorch/FBGEMM. Evaluation configurations and data are released on GitHub and Huggingface (Section 5.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we are releasing the data generated as part of evaluations with publicly available benchmarks which can be found on Huggingface' (Section 5.1). Standard public benchmarks are used for evaluation. However, the pre-training data itself is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While hardware is described in detail (16K H100 GPUs, 80GB HBM3, etc.) in Section 3.3, there are no environment specifications (requirements.txt, Dockerfile, library versions) for reproducing the evaluation pipeline."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides extensive methodological detail but no step-by-step reproduction instructions. Evaluation setup details are deferred to external links ('can be found on our Github repository here') but these are not inline instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence intervals for most benchmark results (Tables 9-14, 17-22), following Madaan et al. (2024b). The CI formula is explicitly given: CI(S) = 1.96 × sqrt(S × (1-S) / N)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Despite reporting confidence intervals, the paper does not perform formal statistical significance tests when claiming one model outperforms another. Comparisons are made by comparing point estimates and CIs but no p-values or tests are used."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported as absolute scores on benchmarks with baseline comparisons throughout (e.g., Table 2 shows Llama 3 405B at 87.3 MMLU vs GPT-4 at 85.1). The magnitude of differences is clear from the reported numbers with baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the number of evaluation examples or human evaluation sample sizes. The human evaluation uses ~7,000 prompts but the choice is not justified with power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports confidence intervals based on the binomial approximation from benchmark size, but does not report variance across multiple experimental runs or seeds. Results appear to be single-run evaluations."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included throughout: GPT-4, GPT-4o, Claude 3.5 Sonnet, Gemini, Mixtral, Mistral, Gemma, Nemotron 4 340B, and Llama 2 models (Tables 2, 9-22)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include the most competitive contemporary models at the time of publication: GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro, and Gemma 2 (July 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple ablation-style analyses are presented: scaling law experiments (Section 3.2.1), data mix experiments (Section 3.1.2), annealing experiments (Section 3.1.3), long-context data mix ablations (Section 4.3.4), and DPO modifications (Section 4.1.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports results across dozens of metrics including MMLU, HumanEval, MBPP, GSM8K, MATH, ARC, GPQA, IFEval, MGSM, pass@1, word error rate, BLEU, and human evaluation win rates."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 describes extensive human evaluations comparing Llama 3 405B against GPT-4, GPT-4o, and Claude 3.5 Sonnet using ~7,000 prompts with 7-point preference ratings across multiple capabilities (Figure 17)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports on standard benchmark test sets and explicitly states the pre-training team was 'strongly incentivized to prevent contamination' (Section 10). Evaluation sets were held out from post-training via exact match decontamination (Section 5.2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down extensively by category: commonsense reasoning, knowledge, reading comprehension, math/reasoning, code, multilingual, tool use, long context (Tables 8-22). Human evaluation results are broken down by capability (Figure 17)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Adversarial benchmarks (Section 5.1.3) show where models struggle. Negative results on code interpreter abuse (10.4% compliance, Section 5.4.5), prompt injection susceptibility (21.7%, Figure 22), and safety violations are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: annealing improvements on 405B were 'negligible' (Section 3.1.3); markdown was found 'harmful' (Section 3.1.1); PPO was found to require more compute and perform worse than DPO (Section 4.1.4); training 405B on its own generated data was 'not helpful (and can even degrade performance)' (Section 4.3.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'comparable quality to leading language models such as GPT-4 on a plethora of tasks.' Table 2 and human evaluations (Figure 17) support this — Llama 3 405B is competitive with but does not uniformly beat GPT-4o and Claude 3.5 Sonnet."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are generally backed by controlled ablations. E.g., scaling law experiments (Section 3.2.1) use controlled compute budgets, data mix experiments test individual variables, and DPO modifications are tested with before/after comparisons. The paper is appropriately cautious in framing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested benchmarks and settings. The paper specifies model sizes (8B, 70B, 405B), specific benchmarks, and settings. Multimodal results are explicitly noted as 'still under development and not yet ready for release.' The paper avoids overly broad claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.4 discusses contamination as an alternative explanation for benchmark performance. Section 5.1.2 examines robustness to prompt format and label order. Section 5.1.3 tests adversarial alternatives. Human evaluation limitations regarding annotator bias are discussed (Section 5.3)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports specific benchmark metrics without inflating them to broader claims. It says 'performs on par with leading language models' based on specific benchmarks, not 'achieves AGI.' Human evaluations are framed as measuring specific capabilities (coding, reasoning, multilingual) rather than general intelligence."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Llama 3 model specifications are given in detail (Table 3: 8B, 70B, 405B parameters, architecture details). Competitor models are identified with specific names (GPT-4 0125 API version, GPT-4o API version, Claude 3.5 Sonnet API version — Section 5.3). Table 2 specifies GPT-4 (0125)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Several actual prompts are provided: the system prompt for steerability (Section 4.3.7), ASR/AST system prompts (Section 8.3.1), video evaluation prompts (Section 7.7). The paper references external repos for full evaluation configurations. The steerability system prompt example is verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Extensive hyperparameters are reported: learning rates (Table 3, Sections 3.4.1, 4.1.3, 4.1.4), batch sizes, DPO β=0.1, NLL loss coefficient 0.2, temperature settings for rejection sampling, RoPE frequency, context lengths, and parallelism configurations (Table 4)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. Models are evaluated directly on benchmarks. Tool use is a capability of the model itself, not an external scaffold."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides extensive detail on data preprocessing: PII filtering, text extraction, URL/document/line-level deduplication, heuristic filtering, model-based quality filtering, knowledge classification, and multilingual processing. Post-training data processing is detailed in Section 4.2.3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4.8 is titled 'Limitations' and discusses safety limitations. Section 5.3 acknowledges limitations of human evaluations. The conclusion (Section 10) notes 'development of high-quality foundation models is still in its infancy.'"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: contamination analysis (Section 5.1.4) identifies benchmarks where contamination may inflate scores (e.g., PiQA, HellaSwag). Section 5.3 notes 'human evaluations can still be influenced by personal biases, backgrounds, and preferences.' Safety limitations note models 'may still generate harmful content.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states multimodal models 'are still under development and not yet ready for release.' Footnote 9 states Llama 3 'has not been optimized or safety tuned for use cases' beyond 8 supported languages. Section 5.4.8 states 'no testing can be guaranteed to be exhaustive.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Pre-training data is not released. Evaluation data generated from public benchmarks is released on Huggingface, but the raw training data, preference annotations, and synthetic data are proprietary."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Pre-training data collection is described in detail (Section 3.1): web crawling, HTML parsing, deduplication methods, quality filtering. Post-training data collection via human annotations is described in Section 4.2.1 with statistics in Tables 6-7."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'human annotators' and 'data vendors' for preference data and safety data, but does not describe how annotators were recruited, their qualifications, compensation, or potential biases from the recruitment process."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in detail: Section 3.1 describes web data curation (PII filtering → text extraction → deduplication → heuristic filtering → model-based filtering). Post-training data pipeline is in Section 4.2.3 (topic classification → quality scoring → difficulty scoring → semantic deduplication)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure or acknowledgments section listing funding sources. The work is clearly done at Meta, but there is no formal funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The paper is clearly attributed to 'Llama Team, AI @ Meta' with the full contributor list in the appendix. All contributors are identified as Meta employees."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta is both the funder and the organization whose product (Llama 3) is being evaluated. Meta has a direct commercial interest in Llama 3 performing well. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Meta employees evaluating Meta's own product have clear financial interests that are not formally declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 'we create our dataset for language model pre-training from a variety of data sources containing knowledge until the end of 2023.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.4 presents a detailed contamination analysis using 8-gram overlap between evaluation sets and the pre-training corpus, following Singh et al. (2024). Table 15 reports contamination percentages and estimated performance gains for each benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.4 explicitly addresses benchmark contamination with quantitative analysis. The organizational design separates the pre-training data team from the evaluation team (Section 10). Annealing data explicitly excludes 'training sets from commonly used benchmarks' (Section 3.1.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The paper includes human evaluations (Section 5.3) and safety studies with human participants (Section 5.4.5 uplift testing), but none are pre-registered."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No IRB or ethics board approval is mentioned for the human evaluation studies or the cybersecurity/biosecurity uplift testing involving human participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation annotator demographics are not reported. For the cybersecurity uplift study, participants are categorized as 'expert' (31) and 'novice' (31) based on offensive security experience, but detailed demographics are absent."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "For the cybersecurity uplift study (Section 5.4.5), participants were 'categorized into expert (31 subjects) and novice (31 subjects) cohorts based on their offensive security experience.' CBRN study participants were 'recruited based on previous experience in relevant areas of scientific or operational expertise.'"
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "The CBRN uplift study describes assignment to 'control' or 'LLM' conditions (Section 5.4.5). The cybersecurity study uses a two-stage within-subjects design where participants complete challenges first without then with LLM assistance."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described for any of the human studies. In the uplift testing, participants clearly know whether they have LLM access. Human evaluation annotators are not blinded to model identity."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No attrition information is provided for any human study. The uplift study starts with 62 volunteers but does not report whether all completed both stages."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 reports inference throughput and latency for both BF16 and FP8 quantization (Figures 24, 27), including tokens/sec and time-to-first-token. MFU is reported in Table 4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states the flagship model was 'pre-trained using 3.8 × 10^25 FLOPs' on 'up to 16K H100 GPUs' (Section 3.3.1). Training stability data covers a 54-day snapshot period (Table 5). BF16 MFU of 38-43% is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results appear to be from single training runs. The paper acknowledges CIs from finite samples but does not report variance across seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs is not explicitly stated. CIs are computed analytically from benchmark size rather than from multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "While scaling law experiments are described (Section 3.2.1), the total hyperparameter search budget for the final model configuration is not stated. The paper mentions 'careful tuning' of parallelism configurations but no search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Scaling law experiments (Section 3.2.1) provide principled selection of model size (402B→405B). Data mix selection uses scaling law experiments. Post-training uses multiple rounds with progressive improvement. Model averaging selects from various configurations (Section 4.1.5)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports results across dozens of benchmarks and makes many comparisons without any correction for multiple comparisons (no Bonferroni, Holm, or similar correction)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Meta evaluates its own Llama 3 models against competitors. While they report reproducing competitor results where possible and selecting the best score, the paper does not explicitly acknowledge the systematic bias of self-evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The scaling law analysis (Section 3.2.1, Figures 2-4) explicitly relates compute budget to performance. The paper discusses training smaller models 'for much longer than is compute-optimal' and the resulting performance tradeoffs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses robustness of benchmarks to design choices (Section 5.1.2), adversarial benchmarks (Section 5.1.3), and contamination analysis (Section 5.1.4). Human evaluations are used as a complement to automated metrics, acknowledging that 'well-designed human evaluations closely reflect the user experience.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Models are evaluated directly on benchmarks without external scaffolding. Tool use evaluation tests the model's native capability. No scaffold confound exists."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper states pre-training data goes 'until the end of 2023' (Section 3.1) and the contamination analysis (Section 5.1.4) addresses temporal overlap between benchmarks and training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of feature leakage (e.g., whether evaluation setup provides hints not available in real usage). The focus is on data contamination rather than feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While deduplication is described for training data, the paper does not discuss whether evaluation examples share structural similarities with training data beyond n-gram overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.4 applies 8-gram overlap analysis following Singh et al. (2024), computing contamination percentages and estimated performance gains for each benchmark (Table 15). Post-training decontamination uses exact match with benchmark prompts."
       }
     }
   },
   "claims": [
     {
       "claim": "Llama 3 405B performs on par with leading language models such as GPT-4 across a variety of tasks.",
       "evidence": "Table 2 shows Llama 3 405B achieves 87.3 MMLU (vs GPT-4 85.1), 89.0 HumanEval (vs GPT-4 86.6), 96.8 GSM8K (vs GPT-4 94.2). Human evaluations (Figure 17) show approximate parity with GPT-4 (0125).",
       "supported": "strong"
     },
     {
       "claim": "Scaling law predictions extrapolated over four orders of magnitude accurately forecast the flagship model's performance.",
       "evidence": "Section 3.2.1 and Figure 4 show the two-step scaling law prediction 'only slightly underestimates the final performance' on ARC Challenge, extrapolating from 10^22 to 3.8×10^25 FLOPs.",
       "supported": "moderate"
     },
     {
       "claim": "Llama 3 8B and 70B outperform competing models of similar sizes on most benchmarks.",
       "evidence": "Tables 9-14 and Figure 12 show Llama 3 8B outperforms Mistral 7B and Gemma 7B on virtually every category. Llama 3 70B outperforms Mixtral 8x22B on most benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "FP8 quantization has very limited impact on model response quality.",
       "evidence": "Figure 26 shows the reward score distribution for FP8 vs BF16 is nearly identical across 100,000 responses. FP8 inference provides up to 50% throughput improvement during pre-fill (Figure 27).",
       "supported": "strong"
     },
     {
       "claim": "Llama 3 release does not provide significant uplift for cybersecurity or CBRNE attacks.",
       "evidence": "Section 5.4.5: cybersecurity uplift study with 62 participants showed 'insignificant uplift.' CBRN study showed 'no significant uplift in performance.' Autonomous attack agents 'fail to effectively use' reconnaissance information.",
       "supported": "moderate"
     },
     {
       "claim": "DPO performed better than PPO for alignment, especially on instruction-following benchmarks.",
       "evidence": "Section 4.1.4 states 'DPO required less compute for large-scale models and performed better, especially on instruction following benchmarks like IFEval.'",
       "supported": "weak"
     },
     {
       "claim": "Training the 405B model on its own generated data is not helpful and can degrade performance.",
       "evidence": "Section 4.3.1 states 'our initial experiments revealed that training Llama 3 405B on its own generated data is not helpful (and can even degrade performance).' No quantitative evidence provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Meta employees evaluate Llama 3 models across all benchmarks. While they reproduce competitor results 'where possible,' the paper does not acknowledge the systematic bias of self-evaluation. Baselines are run through Meta's pipeline, and the 'best score' selection favors their setup."
     },
     {
       "flag": "Selective competitor availability",
       "detail": "For 405B class comparisons, many competitor numbers are unavailable ('it is not possible to (re)compute benchmark values' for comparable models). Category averages are omitted for 405B precisely because not all numbers are available. This makes comprehensive comparison impossible."
     },
     {
       "flag": "Human evaluation opacity",
       "detail": "Human evaluation annotators, their demographics, training, and potential biases are not described. Only a 'small set of researchers who do not contribute to model development' had access (Section 10), but their identities and potential biases are unknown."
     },
     {
       "flag": "Safety benchmarks are internal and non-reproducible",
       "detail": "Section 5.4.4 explicitly acknowledges 'these safety benchmarks are internal to Meta' and results are 'not reproducible externally.' Competitor safety results are anonymized ('Comp. 1', 'Comp. 2', 'Comp. 3'), preventing independent verification."
     },
     {
       "flag": "No seed sensitivity or multi-run analysis",
       "detail": "For a model trained with 3.8×10^25 FLOPs, no seed sensitivity analysis is performed. CIs are derived analytically from benchmark size rather than empirical variance across runs, underestimating true uncertainty."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, a key code generation benchmark used extensively in Llama 3 evaluation."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "Foundational RLHF paper that Llama 3's post-training methodology builds upon."
     },
     {
       "title": "Direct preference optimization: Your language model is secretly a reward model",
       "authors": ["Rafael Rafailov"],
       "year": 2023,
       "relevance": "Core alignment method used in Llama 3 post-training, preferred over PPO for scalability."
     },
     {
       "title": "Constitutional AI: harmlessness from AI feedback",
       "authors": ["Yuntao Bai"],
       "year": 2022,
       "arxiv_id": "2212.08073",
       "relevance": "Influential safety alignment approach; Llama 3's rejection sampling draws from this work."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Direct predecessor to Llama 3; many training recipes are extended from this work."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Code expert training recipe that Llama 3's code capabilities build upon."
     },
     {
       "title": "Toolformer: Language models can teach themselves to use tools",
       "authors": ["Timo Schick"],
       "year": 2024,
       "relevance": "Influential tool-use approach; Llama 3 differs by relying on human annotations rather than self-taught tool use."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws that inform Llama 3's model sizing and compute allocation decisions."
     },
     {
       "title": "Purple llama cyberseceval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Safety evaluation framework used to assess Llama 3's cybersecurity risks."
     },
     {
       "title": "Many-shot jailbreaking",
       "authors": ["Cem Anil"],
       "year": 2024,
       "relevance": "Long-context jailbreaking attack that Llama 3 specifically mitigates in safety finetuning."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws paper that Llama 3's compute-optimal training decisions build upon."
     },
     {
       "title": "Augmented language models: a survey",
       "authors": ["Grégoire Mialon"],
       "year": 2023,
       "arxiv_id": "2302.07842",
       "relevance": "Survey of tool-augmented LLMs relevant to Llama 3's tool use capabilities."
     }
   ]
 }