ai-research-survey

scan.json (30426B)

---

 {
   "paper": {
     "title": "StarCoder: may the source be with you!",
     "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi", "Niklas Muennighoff", "Denis Kocetkov", "Chenghao Mou", "Marc Marone", "Christopher Akiki", "Jia Li", "Jenny Chim", "Qian Liu", "Evgenii Zheltonozhskii", "Terry Yue Zhuo", "Thomas Wang", "Olivier Dehaene", "Mishig Davaadorj", "Joel Lamy-Poirier", "João Monteiro", "Oleh Shliazhko", "Nicolas Gontier", "Nicholas Meade", "Armel Zebaze", "Ming-Ho Yee", "Logesh Kumar Umapathi", "Jian Zhu", "Benjamin Lipkin", "Muhtasham Oblokulov", "Zhiruo Wang", "Rudra Murthy", "Jason Stillerman", "Siva Sankalp Patel", "Dmitry Abulkhanov", "Marco Zocca", "Manan Dey", "Zhihan Zhang", "Nour Fahmy", "Urvashi Bhattacharyya", "Wenhao Yu", "Swayam Singh", "Sasha Luccioni", "Paulo Villegas", "Maxim Kunakov", "Fedor Zhdanov", "Manuel Romero", "Tony Lee", "Nadav Timor", "Jennifer Ding", "Claire Schlesinger", "Hailey Schoelkopf", "Jan Ebert", "Tri Dao", "Mayank Mishra", "Alex Gu", "Jennifer Robinson", "Carolyn Jane Anderson", "Brendan Dolan-Gavitt", "Danish Contractor", "Siva Reddy", "Daniel Fried", "Dzmitry Bahdanau", "Yacine Jernite", "Carlos Muñoz Ferrandis", "Sean Hughes", "Thomas Wolf", "Arjun Guha", "Leandro von Werra", "Harm de Vries"],
     "year": 2023,
     "venue": "Transactions on Machine Learning Research",
     "arxiv_id": "2305.06161"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 states 'open-sourcing all code repositories for building the model on GitHub.' The Code LM Evaluation Harness is also released (Section 6). Models are released on Hugging Face."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The training data is The Stack v1.2 (Kocetkov et al., 2022), which is publicly available. The PII annotation dataset is available under gated access at https://hf.co/BigCode (Section 4.3)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. Hardware is described (512 A100 GPUs, Section 5.6) but software dependencies and library versions are not specified in sufficient detail for reproduction."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions or README with commands are provided in the paper. Training hyperparameters are given but there are no runnable scripts or instructions to replicate the training or evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results throughout (Tables 12-27) are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims ('StarCoder outperforms every open Code LLM') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with absolute scores and baselines in context (e.g., 'StarCoder 33.6 vs code-cushman-001 33.5 on HumanEval', Table 12), allowing readers to assess the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of samples for pass@k estimation (n=200 for open models, n=20 for API models) is stated but not justified. No power analysis or discussion of whether these sample sizes are sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. Table D.3 shows per-problem pass counts vary dramatically across checkpoints (e.g., 5 to 197) but no systematic variance reporting is done for the main results."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines including CodeGen-16B-Multi, CodeGen-16B-Mono, CodeGeeX, code-cushman-001, LLaMA family, PaLM, LaMDA (Tables 12-27)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary models like LLaMA (2023), CodeGeeX (2023), and code-cushman-001. These were state-of-the-art at the time of evaluation."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study examining which architectural choices or training data components contribute to performance. StarCoder vs StarCoderBase shows the effect of Python fine-tuning but no systematic component ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1 on HumanEval/MBPP/DS-1000/MultiPL-E, BLEU for docstring generation, perplexity for long contexts, F1 for PII detection, stereotype/ICAT scores for bias, toxicity scores."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the model's code generation quality. All evaluations are automated (pass@k on test cases, automated metrics). Section 8 has qualitative examples but no systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training data was decontaminated by removing files containing docstrings or solutions from HumanEval, MBPP, APPS, GSM8K, and DS1000 (Section 5.2). Perplexity evaluation used GPL-licensed repos not in the training data (Section 6.4)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns: DS-1000 results by library (Table 13), MultiPL-E results by programming language (Table 15), StereoSet by bias domain (Table 25), PII detection by entity type (Tables 5, 8, 9)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses performance instability for low-resource languages (R's pass@1 drops between checkpoints). Table D.3 shows high variance in per-problem pass rates. Section E.3 discusses empty/pass solutions. Section 6.2.2 discusses security vulnerabilities."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: performance drops for low-resource languages during training (Section 6.3, Figure 2), Ruby and Racket underperformance for StarCoder vs StarCoderBase (Table 15), higher toxicity than CodeGen (Table 26), the prompt that boosts StarCoder hurts CodeGen (Section E.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about outperforming open Code LLMs and matching code-cushman-001 are supported by Tables 12-15. Claims about PII redaction are supported by Section 4. Attribution tool claims are supported by Section 9."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims (e.g., Python fine-tuning 'resulting in the creation of StarCoder' which outperforms on Python) but lacks controlled ablations. The claim that code+NL training contributes to reasoning (Section 7.4: 'we speculate that the mixture of code and natural language...contributes') is acknowledged as speculation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested settings. The paper specifies exact benchmarks and languages tested. Section 10.2 notes English-only evaluations as a limitation. Claims are made per-benchmark rather than about 'code generation' in general."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Alternative explanations for performance differences are not systematically discussed. For example, the paper does not consider whether StarCoder's advantage comes from data quality, data quantity, architecture, or training approach. Section 6.3 notes performance variance but does not explain it."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper appropriately uses benchmark-specific metrics (pass@1, BLEU, F1) without overclaiming broader capabilities like 'programming ability.' Section 6.1.2 explicitly discusses how HumanEval/MBPP are 'simple programming puzzles that are not representative of the code that most programmers write' and uses DS-1000 as a more realistic complement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Model sizes and architectures are specified in detail (Table 11). For external models: CodeGen-16B-Multi, CodeGen-16B-Mono, CodeGeeX-13B, code-cushman-001 (12B) are named with sizes. LLaMA sizes (7B, 13B, 33B, 65B) are specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates are provided in Section 5.1 (training format), Section E.3 (HumanEval prompting), and Appendix F (full technical assistant prompt). The evaluation harness uses standard benchmark prompts which are publicly available."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters detailed in Section 5.5 (learning rate, batch size, Adam parameters, warmup). Evaluation hyperparameters: temperature 0.2 for pass@1, temperature 0.8 for k>1, n=200 samples (Section 6.1.1). DS-1000: temperature=0.2, top_p=0.5, max_length=1024 (Table 13)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. StarCoder is evaluated as a standalone code generation model."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides extensive documentation of data curation: language selection criteria (Section 3.1), visual inspection with 18 annotators (Section 3.1), XML/alpha/HTML/JSON/YAML filters with specific thresholds, deduplication pipeline (Section 3.5), Git commit filters (Table 4), and decontamination (Section 5.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.2 'Limitations' provides substantive discussion covering dataset licensing limitations, opt-out process gaps, PII detection limitations, malicious code risks, model limitations, and English-only evaluations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.2 discusses specific threats: the license detector may have misclassified repositories, PII model has false positives/negatives that vary across languages, duplicate code in other repos can bypass opt-out, potential training data overlap in some evaluations (Tables 18, 20)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 10.2 explicitly states: 'We evaluated the performance of StarCoder solely on English-based benchmarks.' Specific attribution tool limitations are stated: 'will not find matches to code that was not included or that was removed from the dataset.' Code attribution tools 'do not attempt to distinguish between generic code or protected content.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The training data (The Stack v1.2) is publicly available. The PII annotation dataset is available under gated access. Models are publicly released on Hugging Face."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes data collection in extensive detail: source (permissively licensed GitHub repos via The Stack), language selection criteria, filtering procedures, deduplication methods, and data volumes (Tables 1-2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "PII annotation recruitment is described in Section 4.1: 1,399 crowd-workers from 35 countries via Toloka platform, compensation of $7.30/hour based on purchasing power parity analysis, average 3.1 hours worked. Data inspection used 18 community annotators (Section 3.1)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with volumes at each stage: Tables 1-2 show file counts and volumes after deduplication and after filtering. Section 3.3 describes GitHub issues filtering with volume removed at each step (18%, 17%, 14%). Git commit filters in Table 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section states: 'We would thank Hugging Face for providing the compute resources to train the StarCoder models.' NSF grants SES-2326174 and CCF-2102288 and Adams Fellowship are also disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed in detail: Hugging Face, ServiceNow Research, multiple universities and research labs. The project is co-stewarded by two industry research labs (Section 1)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Hugging Face provided compute and many authors are affiliated with Hugging Face. Hugging Face hosts The Stack dataset and the StarCoder models, giving them a commercial interest in the models performing well."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided. Several authors are from companies (Hugging Face, ServiceNow, IBM Research, etc.) that could benefit commercially from StarCoder's success."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state a specific training data cutoff date. The Stack v1.2 is referenced but no date range for when the GitHub data was collected is provided in this paper."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes explicit decontamination: removing files containing docstrings/solutions from HumanEval, MBPP, APPS, GSM8K, and DS1000. The paper also acknowledges potential overlap for type prediction and docstring benchmarks (Tables 18, 20)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 directly addresses contamination by decontaminating the training data against HumanEval, MBPP, APPS, GSM8K, and DS1000. The paper notes that some evaluation datasets (type prediction, docstring generation) 'may overlap with the training data' (Tables 18, 20)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. The PII annotation is data collection, not a study of human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study requiring IRB approval."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study. Crowd-worker demographics (countries) are reported for the annotation task but this is data collection, not a human study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for StarCoder despite evaluating it extensively across benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6: 512 A100 80GB GPUs across 64 nodes. Section 5.7: 320,256 GPU-hours for StarCoderBase, 16.68 tonnes CO2eq. StarCoder fine-tuning adds 3.5% training time (0.58 tonnes CO2eq). PII detection: 800 GPU-hours (Section 4.3)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results reported across multiple random seeds. Table D.3 shows dramatic per-problem variance across training checkpoints for R but this is training dynamics, not seed sensitivity of the final model evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1.1 states n=200 samples for open models, n=20 for API models. DS-1000 reports 'mean pass@1 accuracy averaged over 40 samples' (Table 13). Security benchmark: 25 completions per scenario (Section 6.2.2)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The training hyperparameters appear chosen but no search procedure or budget is described."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the specific architecture size (15.5B) or training configuration was chosen. No comparison of configurations tried."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed, so no multiple comparison correction either. The paper compares across 19 languages and many benchmarks without any correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors developed StarCoder and evaluate it against baselines using their own evaluation harness (Code LM Evaluation Harness, Section 6). No acknowledgment of self-comparison bias. However, they do use standard benchmarks which mitigates this somewhat."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison at matched compute budgets. StarCoder (15.5B) is compared against models of very different sizes (1.1B to 540B) without normalizing for compute. Table D.1 shows code-davinci-002 (175B) substantially outperforms StarCoderBase but compute difference is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1.2 explicitly discusses construct validity: 'A major limitation of HumanEval and MBPP is that they are simple programming puzzles that are not representative of the code that most programmers write.' The paper uses DS-1000 as a more realistic complement and shows HumanEval/MBPP performance 'does not always correlate' with DS-1000 performance."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated as standalone code generators."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes decontamination by removing files containing docstrings/solutions from evaluation benchmarks. The paper explicitly addresses temporal concerns by noting when benchmarks were published relative to data collection."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit discussion of feature leakage. The evaluation setup (providing function signatures and docstrings) is standard but not analyzed for potential leakage of answer information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 describes near-deduplication (MinHash with Jaccard similarity 0.7) to address duplicate data. The paper acknowledges potential overlap between evaluation datasets and training data for type prediction and docstring benchmarks (Tables 18, 20)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes a concrete decontamination pipeline: removing files containing docstrings/solutions from HumanEval, MBPP, APPS, GSM8K, and DS1000. Section 9.1 describes a Bloom-filter-based membership inference tool for post-hoc detection."
       }
     }
   },
   "claims": [
     {
       "claim": "StarCoder outperforms every open Code LLM that supports multiple programming languages on HumanEval and MBPP.",
       "evidence": "Table 12: StarCoder achieves 33.6% pass@1 on HumanEval and 52.7% on MBPP, exceeding all open-access models listed. StarCoderBase achieves 30.4% and 49.0% respectively.",
       "supported": "strong"
     },
     {
       "claim": "StarCoder matches or outperforms OpenAI's code-cushman-001 model.",
       "evidence": "Table 12: StarCoder 33.6% vs code-cushman-001 33.5% on HumanEval; 52.7% vs 45.9% on MBPP. Table 13: StarCoder outperforms on DS-1000 overall (26.0% vs 18.1%).",
       "supported": "strong"
     },
     {
       "claim": "StarCoder substantially outperforms all other models on DS-1000 data science benchmarks.",
       "evidence": "Table 13: StarCoder achieves 26.0% overall vs code-cushman-001's 18.1% and CodeGen-16B-Mono's 11.7%. Improvement is consistent across all 7 library categories.",
       "supported": "strong"
     },
     {
       "claim": "StarCoderBase outperforms every open-access model on all 19 programming languages in MultiPL-E.",
       "evidence": "Table 15: StarCoderBase leads or is competitive on most languages. However, CodeGen-16B-Multi outperforms on Ruby (0.00 vs 17.25 but this seems like an error as StarCoder gets only 1.24), suggesting some nuance.",
       "supported": "moderate"
     },
     {
       "claim": "The PII detection model outperforms regex baselines on email, IP address, and key detection.",
       "evidence": "Table 8: NER + pseudo labels achieves F1 of 98.15% (email), 91.94% (IP), 70.41% (key) vs regex baseline of 96.83%, 78.65%, 6.74%.",
       "supported": "strong"
     },
     {
       "claim": "StarCoderBase shows lower social bias than LLaMA-13B and CodeGen-16B-Multi.",
       "evidence": "Table 25: StarCoderBase overall stereotype score 55.53% (closest to ideal 50%) vs LLaMA-13B 63.40% and CodeGen 61.29%. ICAT score 76.65% vs 64.14% and 67.55%.",
       "supported": "moderate"
     },
     {
       "claim": "Model performance on HumanEval/MBPP does not always correlate with performance on DS-1000.",
       "evidence": "Section 6.1.2: CodeGen-Mono slightly outperforms code-cushman-001 on HumanEval/MBPP but is significantly worse on DS-1000 (11.7% vs 18.1%).",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StarCoder (15.5B parameters) is the highest-performing open-access code LLM at time of publication, matching or outperforming the closed-access code-cushman-001 across HumanEval, MBPP, and DS-1000 benchmarks. The paper introduces a comprehensive PII redaction pipeline with a custom encoder model achieving >90% F1 on most entity types, and novel attribution tools (Bloom filter membership checking and BM25 search index). StarCoderBase shows lower social bias than comparably-sized models and benefits substantially from its 8K context window. Performance on simple benchmarks (HumanEval/MBPP) does not correlate well with more realistic data science tasks (DS-1000).",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All performance comparisons across all benchmarks are raw number comparisons without any statistical tests. Claims like 'outperforms every open Code LLM' rest on comparing point estimates without uncertainty quantification."
     },
     {
       "flag": "Potential self-evaluation bias",
       "detail": "The authors developed the model, the evaluation harness (Code LM Evaluation Harness, Section 6), and ran all evaluations. No independent evaluation is included."
     },
     {
       "flag": "Non-independent funder",
       "detail": "Hugging Face provided compute resources and many authors are Hugging Face employees. Hugging Face commercially hosts the model and dataset, creating a financial interest in positive results."
     },
     {
       "flag": "Missing variance reporting",
       "detail": "Table D.3 reveals dramatic variance in pass rates across training checkpoints (e.g., 5 to 197 out of 200 for the same problem), yet no systematic variance reporting is done for the final model's evaluation results."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval benchmark and Codex, the foundational code generation evaluation framework."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "relevance": "Introduces MBPP benchmark for evaluating code generation from natural language descriptions."
     },
     {
       "title": "SantaCoder: don't reach for the stars!",
       "authors": ["Loubna Ben Allal", "Raymond Li", "Denis Kocetkov"],
       "year": 2023,
       "relevance": "Predecessor open code model from BigCode community; establishes architecture and training approaches built upon by StarCoder."
     },
     {
       "title": "MultiPL-E: a scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "arxiv_id": "2208.08227",
       "relevance": "Multi-language code generation benchmark translating HumanEval/MBPP to 18 languages."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Security benchmark for code generation models measuring vulnerability rates."
     },
     {
       "title": "DS-1000: a natural and reliable benchmark for data science code generation",
       "authors": ["Yuhang Lai", "Chengxi Li", "Yiming Wang"],
       "year": 2022,
       "relevance": "Realistic data science code generation benchmark addressing limitations of HumanEval/MBPP."
     },
     {
       "title": "CodeGen: an open large language model for code with multi-turn program synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2023,
       "relevance": "Key baseline open code LLM (16B parameters) for multi-language code generation."
     },
     {
       "title": "The Stack: 3 TB of permissively licensed source code",
       "authors": ["Denis Kocetkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2022,
       "arxiv_id": "2211.15533",
       "relevance": "The training dataset for StarCoder with data governance framework for code LLM training."
     },
     {
       "title": "InCoder: a generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2022,
       "relevance": "Fill-in-the-middle code model, key baseline for infilling evaluation."
     },
     {
       "title": "Holistic evaluation of language models",
       "authors": ["Percy Liang", "Rishi Bommasani", "Tony Lee"],
       "year": 2022,
       "relevance": "HELM benchmark suite used for evaluating StarCoder on natural language reasoning tasks."
     },
     {
       "title": "PAL: Program-aided language models",
       "authors": ["Luyu Gao", "Aman Madaan", "Shuyan Zhou"],
       "year": 2022,
       "relevance": "Program-aided reasoning approach used to evaluate StarCoder's mathematical reasoning capabilities."
     },
     {
       "title": "The gradient of generative AI release: Methods and considerations",
       "authors": ["Irene Solaiman"],
       "year": 2023,
       "relevance": "Framework for responsible AI model release that informed StarCoder's release strategy."
     }
   ]
 }