ai-research-survey

scan.json (29951B)

---

 {
   "paper": {
     "title": "Code Llama: Open Foundation Models for Code",
     "authors": [
       "Baptiste Rozière",
       "Jonas Gehring",
       "Fabian Gloeckle",
       "Sten Sootla",
       "Itai Gat",
       "Xiaoqing Ellen Tan",
       "Yossi Adi",
       "Jingyu Liu",
       "Romain Sauvestre",
       "Tal Remez",
       "Jérémy Rapin",
       "Artyom Kozhevnikov",
       "Ivan Evtimov",
       "Joanna Bitton",
       "Manish Bhatt",
       "Cristian Canton Ferrer",
       "Aaron Grattafiori",
       "Wenhan Xiong",
       "Alexandre Défossez",
       "Jade Copet",
       "Faisal Azhar",
       "Hugo Touvron",
       "Louis Martin",
       "Nicolas Usunier",
       "Thomas Scialom",
       "Gabriel Synnaeve"
     ],
     "year": 2023,
     "venue": "arXiv",
     "arxiv_id": "2308.12950"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL: https://github.com/facebookresearch/codellama (footnote 1, Section 1). They also state: 'We provide inference code for both completion and infilling models in the accompanying repository.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The training dataset is described (Table 1) as publicly available code with sampling proportions, but the actual curated dataset is not released. The paper states the dataset consists of 'publicly available code' but does not provide a download link or archive for their specific data mix."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided in the paper. The paper describes optimizer settings and training details but not the software environment needed to reproduce the work."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides inference code via the GitHub repository, but no step-by-step instructions for reproducing the training pipeline (data preparation, training scripts, fine-tuning stages) are provided. The training details in Section 2.6 are at a high level."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '67% on HumanEval'). No confidence intervals, error bars, or uncertainty quantification is provided for any of the benchmark results in Tables 2-9."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'Code Llama - Python 7B outperforms Llama 2 70B') based solely on comparing point estimates without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage point improvements with baseline context. For example, Section 3.1.1: 'training on 100B extra tokens of a Python-heavy data mix leads to significant gains... between 4.3% points and 8.3% points in HumanEval pass@1 and between 1.2% points and 6.4% points in MBPP pass@1.' Tables provide full baseline and model scores enabling effect size computation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sizes of the evaluation benchmarks used (HumanEval has 164 problems, MBPP has ~500). No power analysis or discussion of whether these sample sizes are sufficient for the claims being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or spread measures are reported across runs. Pass@1 scores use greedy decoding (single deterministic run). Pass@10 and pass@100 use sampling but only report unbiased estimators without variance across independent evaluation runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: Llama 2 (all sizes), StarCoder, CodeGen-Multi, CodeGeeX, code-cushman-001, GPT-3.5, GPT-4, PaLM, PaLM-Coder, PaLM 2-S, Falcon, MPT, and more. See Tables 2, 4, 9."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models as of 2023: GPT-4 (2023), PaLM 2-S (2023), StarCoder (2023), GPT-3.5/ChatGPT. These represent the state of the art at the time of publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Extensive ablation studies are presented: (i) fine-tuning Llama 2 vs. training from scratch (Section 3.4.1, Figure 5b), (ii) impact of infilling training on downstream tasks (Table 5), (iii) effect of long context fine-tuning (Tables 7, 10), (iv) self-instruct data value (Table 8), (v) temperature effects on pass@k (Figure 6), (vi) RoPE frequency ablations (Table 18)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@10, pass@100 across HumanEval, MBPP, APPS, MultiPL-E. Long context evaluations use perplexity, exact match, and BLEU (Table 7). Safety evaluations use TruthfulQA, ToxiGen, and BOLD (Table 9)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Red teaming with 25 Meta employees including domain experts in responsible AI, malware development, and offensive security engineering (Section 4, 'Red teaming'). Participants provided qualitative assessments and generated adversarial prompts. This constitutes human evaluation of the model's safety outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval, MBPP, APPS, and MultiPL-E are standard held-out test benchmarks. The paper explicitly notes that the LCC data 'are included in our code training data' (footnote 2, Section 3.3), showing awareness of the distinction, and resamples a separate 'LCC-balanced' set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Breakdowns are provided: per-language results in MultiPL-E (Table 4), per-difficulty in APPS (Table 3: introductory/interview/competition), per-demographic in safety benchmarks (Tables 20-25), per-file-length in LCC (Figure 9b), per-model-size throughout."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Several failure modes are discussed: (1) degradation from LCFT on short sequences (Section 3.3), (2) false refusals in safety (Section 4, 'False refusals'), (3) SPM format failure in random span infilling (Appendix E), (4) 7B model's poor retrieval for keys at the beginning of prompts (Section 3.3, Table 17)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: (1) LCFT 'slightly hurts performance on standard code synthesis benchmarks' with quantified decreases (Section 3.3), (2) infilling training incurs 'a small cost on HumanEval and MBPP pass@k metrics' (Table 5), (3) instruction fine-tuning 'somewhat distillate Llama 2-Chat' at the cost of code generation performance (Section 6), (4) Code Llama - Python performs worse than Code Llama on introductory APPS problems (Section 3.1.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'state-of-the-art performance among open models' is supported by Tables 2 and 4; 'scores of up to 67% and 65% on HumanEval and MBPP' matches Table 2 (Code Llama - Instruct 70B: 67.8% HumanEval, Code Llama - Python 70B: 65.6% MBPP); 'Code Llama - Python 7B outperforms Llama 2 70B' is supported in Table 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about the effect of training stages are supported by controlled ablations: FIM vs. no FIM (Table 5), LCFT vs. no LCFT (Tables 7, 10), self-instruct vs. no self-instruct (Table 8), Llama 2 init vs. training from scratch (Figure 5b). Each ablation manipulates a single variable while holding others constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Open Foundation Models for Code' and abstract claims of 'state-of-the-art performance' are broad. While the paper tests on multiple benchmarks and languages, it does not bound claims to the tested settings. The abstract says 'all our models outperform every other publicly available model on MultiPL-E' without qualifying which languages/tasks were tested. The paper does not discuss limitations of benchmark-based evaluation for real-world code generation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for the observed improvements. For example, it does not consider whether improvements from Llama 2 initialization could be due to data overlap between Llama 2's training data and the code benchmarks, or whether the self-instruct gains could be partially explained by the filtering mechanism rather than the data quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model sizes (7B, 13B, 34B, 70B), the base model (Llama 2), and for external comparisons includes specific versions: 'gpt-3.5-turbo-16k-0613' (Section 3.3), 'code-cushman-001' (Table 2). The Code Llama models themselves are the subject of the paper with clear specifications."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix H: self-training question generation (Figure 10), unit test generation (Figure 11), solution generation (Figure 12), MBPP zero-shot evaluation (Figure 13), and APPS evaluation prompts (Figure 14). While these contain placeholders, the fill values are described (e.g., questions from the generation step)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed hyperparameters in Section 2.6: optimizer (AdamW, beta1=0.9, beta2=0.95), cosine schedule with 1000 warmup steps, batch size 4M tokens, learning rates (3e-4, 1.5e-4), LCFT learning rate 2e-5, RoPE base period theta=10^6. Evaluation: temperature, nucleus sampling p=0.95 specified per benchmark."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Code Llama models are standard language models for code generation and infilling without tool use, feedback loops, or agent architectures."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents the dataset: near-deduplicated publicly available code (85%), natural language related to code (8%), natural language (7%), with BPE tokenization. Table 1 provides sampling proportions, epochs, and disk sizes. The self-instruct pipeline (Section 2.5) documents each step: 62,000 questions generated, deduped to ~52,000, filtered by unit test execution to ~14,000 triplets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated 'Limitations' or 'Threats to Validity' section. The Discussion (Section 6) mentions some tradeoffs (LCFT and infilling cost on benchmarks, safety vs. coding performance) but these are framed as design decisions rather than limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address potential issues such as benchmark overfitting, data contamination, the representativeness of HumanEval/MBPP for real-world coding tasks, or limitations of automated safety benchmarks."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of what populations, tasks, or settings are excluded from the claims. The Discussion (Section 6) mentions 'Further work is needed for LLMs to understand context and nuance in their instructions' but does not systematically bound the scope of results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the training data nor the raw evaluation outputs are made available. Only aggregated benchmark scores are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the training data collection: publicly available code (near-deduplicated), natural language datasets related to code, sampling proportions, and disk sizes (Table 1). The self-instruct pipeline (Section 2.5) describes generation of 62,000 questions using Llama 2 70B, deduplication, unit test generation and filtering."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the red teaming exercise (25 Meta employees), the paper does not describe how participants were recruited or selected beyond stating they include 'domain experts in responsible AI, malware development, and offensive security engineering.' Selection criteria and potential selection bias are not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The training pipeline is documented across Sections 2.1-2.6 and Figures 2, 8: Llama 2 base → code training (500B/1T tokens) → infilling training → long context fine-tuning → Python specialization and/or instruction fine-tuning. The self-instruct pipeline documents each stage with counts (62,000 → ~52,000 → ~14,000)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding disclosure or acknowledgment of funding sources is present. The paper is from Meta AI but does not include a standard funding statement."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed with 'Meta AI' affiliation on the first page. One author has a dual affiliation with 'CERMICS École des Ponts ParisTech' and another with 'Hebrew University of Jerusalem.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta AI funded this research and has a direct commercial interest in Code Llama performing well, as it is released under Meta's Llama 2 license for both research and commercial use. The funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. As Meta employees releasing a Meta product, the authors clearly have financial interests related to the findings, but these are not declared."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff date is stated for either Llama 2 or Code Llama. The paper does not specify when the training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly notes that 'LCC data points are included in our code training data' (footnote 2, Section 3.3). For HumanEval infilling, it mentions potential overlap: 'both our models and the models from Allal et al. (2023) and Li et al. (2023) have been trained on datasets that may have an overlap with this evaluation dataset' (Appendix E). However, no analysis of overlap with HumanEval or MBPP benchmarks is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes the LCC overlap, it does not address contamination risk for HumanEval (published 2021), MBPP (published 2021), or APPS (published 2021) — all publicly available before the model's likely training cutoff. The paper uses these as primary benchmarks without analyzing whether solutions appeared in the training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The red teaming exercise is a qualitative security evaluation, not a formal human subjects study. Pre-registration does not apply to adversarial red teaming."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "The red teaming was conducted by Meta employees as part of responsible AI evaluation, not as a formal human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "The red teaming exercise involved Meta employees, not external human participants in a research study. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "The red teaming exercise was an internal security evaluation with domain experts, not a human subjects study requiring formal inclusion/exclusion criteria."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The red teaming is not an experimental study with conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The red teaming is not an experimental study requiring blinding."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable. The red teaming is not a longitudinal human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens-per-second metrics are reported for any model size. The paper does not discuss the practical cost of running Code Llama models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, total compute cost, or hardware specifications are provided. The paper states training token counts (500B, 1T) but does not report the computational resources required."
       }
     }
   },
   "claims": [
     {
       "claim": "Code Llama reaches state-of-the-art performance among open models on several code benchmarks, with scores of up to 67% and 65% on HumanEval and MBPP, respectively.",
       "evidence": "Table 2 shows Code Llama - Instruct 70B achieves 67.8% on HumanEval pass@1 and Code Llama - Python 70B achieves 65.6% on MBPP pass@1, exceeding all other open models listed (StarCoder, CodeGen-Multi, etc.).",
       "supported": "strong"
     },
     {
       "claim": "Code Llama - Python 7B outperforms Llama 2 70B on HumanEval and MBPP.",
       "evidence": "Table 2: Code Llama - Python 7B achieves 38.4% HumanEval pass@1 vs. Llama 2 70B at 30.5%, and 47.6% MBPP pass@1 vs. Llama 2 70B at 45.4%.",
       "supported": "strong"
     },
     {
       "claim": "All Code Llama models outperform every other publicly available model on MultiPL-E.",
       "evidence": "Table 4 shows Code Llama 7B (average 26.3%) outperforms StarCoder Python 15.5B (25.3%), CodeGen-Multi 16B (13.4%), and other baselines on the multilingual average. Larger Code Llama models show even larger margins.",
       "supported": "strong"
     },
     {
       "claim": "Infilling training incurs only a small cost on autoregressive code generation performance.",
       "evidence": "Table 5: FIM training causes 0.6pp average decrease for 7B and 1.1pp for 13B across HumanEval and MBPP metrics, with no cost on autoregressive test set loss.",
       "supported": "strong"
     },
     {
       "claim": "Long context fine-tuning enables stable behavior on sequences of up to 100,000 tokens.",
       "evidence": "Figure 4a shows perplexity decreasing steadily beyond 16K tokens up to 100K with only slight increase after. Table 7 shows LCFT models outperform non-LCFT models on long-context code completion. However, 'stable behavior' is a soft claim — key retrieval accuracy degrades beyond 16K tokens (Table 17).",
       "supported": "moderate"
     },
     {
       "claim": "Initializing from Llama 2 outperforms training from scratch on code for a given compute budget.",
       "evidence": "Section 3.4.1 and Figure 5b: the scratch-trained 7B model at 500B tokens has loss equivalent to Code Llama 7B at ~250B tokens, showing 'the loss of the model trained from scratch is equal to the loss of Code Llama 7B at about half of its training (with 240B less training tokens).'",
       "supported": "strong"
     },
     {
       "claim": "Self-instruct data improves coding benchmark scores and training reliability for instruction-following models.",
       "evidence": "Table 8: self-instruct improves HumanEval pass@1 from 30.5% to 34.8% (7B) and 40.9% to 42.7% (13B). For MBPP zero-shot, it dramatically improves the 13B model from 20.4% to 40.2%, which the paper attributes to more reliable format learning.",
       "supported": "strong"
     },
     {
       "claim": "Code Llama - Instruct is safer than ChatGPT based on safety reward model scores.",
       "evidence": "Figure 7 shows the KDE plot of Llama 2 70B safety reward model scores where Code Llama - Instruct models have more weight in the safer part of the distribution compared to GPT-3.5 Turbo. However, this is based on a single reward model from Meta (potential bias) on a specific set of red-team prompts.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Code Llama is a family of code-specialized LLMs (7B to 70B parameters) derived from Llama 2, achieving state-of-the-art performance among open models on HumanEval (67.8%), MBPP (65.6%), and MultiPL-E benchmarks. The paper demonstrates that initializing from a general-purpose LLM and progressively specializing through code training, infilling, long context fine-tuning, and instruction tuning yields large gains, with infilling and LCFT incurring only modest costs on standard benchmarks. Notably, Code Llama - Python 7B outperforms Llama 2 70B on Python code generation, demonstrating the value of domain specialization over scale alone.",
   "red_flags": [
     {
       "flag": "Company evaluating own product",
       "detail": "All authors are Meta AI employees evaluating Meta's Code Llama models. The paper was released alongside the commercial product launch. While affiliations are disclosed, no competing interests statement is included, and the safety comparison (Figure 7) uses Meta's own Llama 2 safety reward model as the evaluation metric."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All benchmark results are reported as point estimates without confidence intervals, error bars, or variance across runs. For pass@1 with greedy decoding, this is a single deterministic run. For sampling-based metrics (pass@10, pass@100), the unbiased estimator is computed but variance of the estimator is not reported. Small differences between models could be within noise."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "HumanEval, MBPP, and APPS were all publicly available before Code Llama's training. The paper explicitly notes LCC overlap with training data (footnote 2) but does not analyze whether HumanEval or MBPP solutions appeared in the code training corpus. The paper acknowledges potential overlap for the CodeXGLUE infilling benchmark (Appendix E) but takes no action."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations section. Known tradeoffs (LCFT cost, infilling cost) are discussed as engineering decisions rather than limitations. No discussion of benchmark representativeness, potential data contamination, or the gap between benchmark performance and real-world coding utility."
     },
     {
       "flag": "No compute budget disclosed",
       "detail": "Despite training 12 models across 4 sizes with multiple training stages totaling hundreds of billions to a trillion tokens, no GPU hours, hardware specifications, or energy consumption figures are reported. This makes the work unreproducible for resource-constrained researchers."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, the foundational code LLM paper and primary evaluation benchmark used in this work."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Base model that Code Llama is built upon; provides the foundation model, instruction tuning data, and safety alignment approach."
     },
     {
       "title": "StarCoder: May the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Primary open-source code LLM baseline; demonstrates open data curation practices for code models."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David H. Choi", "Junyoung Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Key code generation baseline using massive sampling and filtering; evaluated on competitive programming tasks."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "State-of-the-art closed-source LLM baseline for code generation and general language understanding."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2023,
       "relevance": "Introduces causal masking for code infilling, a key training technique adopted in Code Llama."
     },
     {
       "title": "Efficient training of language models to fill in the middle",
       "authors": ["Mohammad Bavarian", "Heewoo Jun", "Nikolas Tezak"],
       "year": 2022,
       "arxiv_id": "2207.14255",
       "relevance": "Foundational fill-in-the-middle (FIM) training methodology directly used in Code Llama's infilling objective."
     },
     {
       "title": "Extending context window of large language models via positional interpolation",
       "authors": ["Shouyuan Chen", "Sherman Wong", "Liangjian Chen", "Yuandong Tian"],
       "year": 2023,
       "arxiv_id": "2306.15595",
       "relevance": "Related approach to extending LLM context length via RoPE modification, compared against Code Llama's approach."
     },
     {
       "title": "MultiPL-E: A scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "relevance": "Multilingual code generation benchmark used as a primary evaluation tool for Code Llama across 7 programming languages."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, one of the two primary Python code generation benchmarks used to evaluate Code Llama."
     },
     {
       "title": "Textbooks are all you need",
       "authors": ["Suriya Gunasekar", "Yi Zhang", "Jyoti Aneja"],
       "year": 2023,
       "arxiv_id": "2306.11644",
       "relevance": "Demonstrates that high-quality filtered training data can match larger models on code benchmarks; contrasted with Code Llama's approach."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "RLHF methodology that underlies the instruction fine-tuning and safety alignment approach inherited from Llama 2."
     }
   ]
 }