ai-research-survey

scan.json (31881B)

---

 {
   "paper": {
     "title": "StarCoder 2 and The Stack v2: The Next Generation",
     "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal", "Federico Cassano", "Joel Lamy-Poirier", "Nouamane Tazi", "Ao Tang", "Dmytro Pykhtar", "Jiawei Liu", "Yuxiang Wei", "Tianyang Liu", "Max Tian", "Denis Kocetkov", "Arthur Zucker", "Younes Belkada", "Zijian Wang", "Qian Liu", "Dmitry Abulkhanov", "Indraneil Paul", "Zhuang Li", "Wen-Ding Li", "Megan Risdal", "Jia Li", "Jian Zhu", "Terry Yue Zhuo", "Evgenii Zheltonozhskii", "Nii Osae Osae Dade", "Wenhao Yu", "Lucas Krauß", "Naman Jain", "Yixuan Su", "Xuanli He", "Manan Dey", "Edoardo Abati", "Yekun Chai", "Niklas Muennighoff", "Xiangru Tang", "Muhtasham Oblokulov", "Christopher Akiki", "Marc Marone", "Chenghao Mou", "Mayank Mishra", "Alex Gu", "Binyuan Hui", "Tri Dao", "Armel Zebaze", "Olivier Dehaene", "Nicolas Patry", "Canwen Xu", "Julian McAuley", "Han Hu", "Torsten Scholak", "Sebastien Paquet", "Jennifer Robinson", "Carolyn Jane Anderson", "Nicolas Chapados", "Mostofa Patwary", "Nima Tajbakhsh", "Yacine Jernite", "Carlos Muñoz Ferrandis", "Lingming Zhang", "Sean Hughes", "Thomas Wolf", "Arjun Guha", "Leandro von Werra", "Harm de Vries"],
     "year": 2024,
     "venue": "TMLR (under review)",
     "arxiv_id": "2402.19173"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper releases model weights under an OpenRAIL license, the training dataset (The Stack v2) on Hugging Face, search tools, and references multiple GitHub repositories and Hugging Face spaces for evaluation, data inspection, and attribution."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Stack v2 dataset is released via Hugging Face (bigcode/the-stack-v2). SWHIDs for source code are provided for full traceability. The training data composition is fully documented."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While hardware is mentioned (A100 SXM4 80GB, H100) and some training framework details given (FlashAttention-2), there is no requirements.txt, Dockerfile, or detailed environment specification with library versions for reproducing training or evaluation."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology in detail but does not provide runnable scripts or a README with commands to replicate training or evaluation."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 'Pass@1' scores) throughout all tables (Tables 9-18) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims ('outperforms', 'significantly outperforms') but no statistical significance tests are performed. Comparisons are based solely on point estimate differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baselines, e.g., 'StarCoder2-3B outperforms StarCoderBase-3B, exhibiting improvements of 60.2% on HumanEval+ and 32.4% on MBPP+' (§7.1.1). Absolute scores and relative gains are consistently provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of samples used in evaluation (e.g., 50 samples for MultiPL-E, 40 for DS-1000, 20 for HumanEvalFix). These choices follow prior work but are not independently justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The CRUXEval section mentions noise margins (~0.2% from sampling, ~1.5% from benchmark selection) from the original paper, but StarCoder2's own results report no standard deviation or variance across runs. Single-run or unreported-run results throughout."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines are included: StarCoderBase (3B/7B/15B), CodeLlama (7B/13B/34B), DeepSeekCoder (1.3B/6.7B/33B), StableCode-3B, and OctoCoder across all evaluations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include DeepSeekCoder (2024), CodeLlama (2023), and StableCode (2024), which were contemporary and competitive at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is presented. The paper does not systematically vary components (e.g., data sources, FIM rate, repository-context training, IR data) to measure their individual contributions. The Issue vs. Instruct prompt comparison (§7.2) is the closest, but it is not a systematic ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Pass@1, Pass@5, exact match, edit similarity, CodeBLEU, identifier match F1, and accuracy across different benchmarks and tasks."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code quality is included. All evaluation is automated via test suites, exact match, or classifier-based metrics. The security benchmark uses automated CWE detection, not human review."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established held-out benchmarks (HumanEval, MBPP, DS-1000, etc.) and performs decontamination (§3.3) to remove benchmark examples from training data. RepoBench v1.1 explicitly removes duplicates against The Stack v2."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns: MultiPL-E results per programming language (Table 10), DS-1000 per library (Table 11), HumanEvalFix per language (Table 12), BOLD per demographic category (Table 20), CrossCodeEval per language (Table 18)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses StarCoder2-15B's poor FIM performance due to an implementation bug (§7.5), StarCoder2-7B's unexpectedly weak performance relative to its size (§1), and C++ underperformance with the Issue prompt due to incomplete code generation (§7.2.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: StarCoder2-7B underperforms DeepSeekCoder-6.7B significantly; StarCoder2-15B FIM is poor due to a bug; StarCoder2-7B and 15B have high insecure code generation rates (39.8% and 39.2%); the authors state 'It is not clear to this report's authors why StarCoder2-7B does not perform as well.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims StarCoder2-3B outperforms similar models and StarCoderBase-15B (supported by Tables 9-10), StarCoder2-15B matches or outperforms CodeLlama-34B (supported by multiple tables), and DeepSeekCoder-33B is best at code completion for high-resource languages (acknowledged in results). All claims are supported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal claims about training data and design choices (e.g., 'pre-training on pull requests is a viable alternative to pre-training on commits', 'repository-level training consistently outperforms') but these are confounded — multiple variables change simultaneously (data size, composition, architecture, tokenizer) with no controlled ablation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to tested settings. The paper specifies model sizes, benchmarks, and languages tested. It acknowledges limitations like StarCoder2-7B's underperformance and notes DeepSeekCoder-33B dominance on high-resource languages. The title ('StarCoder 2 and The Stack v2') is appropriately scoped."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for performance differences. When StarCoder2-15B outperforms CodeLlama-34B, no discussion of whether this is due to data quality, data quantity, architecture, training recipe, or other factors. The unexplained StarCoder2-7B underperformance is noted but not analyzed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper primarily reports benchmark-specific metrics (Pass@1, exact match, etc.) without overclaiming broader capabilities. It does not conflate benchmark performance with general 'coding ability.' Claims are tied to specific benchmarks and tasks."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "All model versions are specified with exact sizes (StarCoder2-3B/7B/15B, CodeLlama-7B/13B/34B, DeepSeekCoder-1.3B/6.7B/33B, StableCode-3B). Architecture details, hyperparameters, and training configurations are fully documented in Tables 6-8."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The Issue prompt template for HumanEvalFix is provided in §7.2.1 with the full format. Data formatting templates are extensively documented in §5 with sentinel tokens. The evaluation setup follows standard benchmark prompts with references to original papers."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Training hyperparameters are fully reported in Tables 7-8 (learning rate, batch size, iterations, epochs, RoPE θ, optimizer settings). Evaluation hyperparameters (temperature, top-p, number of samples) are stated for each benchmark."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates base language models on standard benchmarks without any agent framework, tool use, or multi-step reasoning pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Extensive documentation of data preprocessing in §2-3: license detection (Figure 1), language filtering, basic filters (line length, autogenerated, alpha), deduplication parameters (5-grams, Jaccard 0.7), PII redaction, decontamination, malware removal, opt-out handling. Table 3 summarizes which steps apply to each source."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 ('Social Impact and Limitations') includes §9.3 ('Challenges and Risks') discussing openness/safety risks, privacy, security, societal bias, representation bias, traceability challenges, and job augmentation vs. automation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: FIM implementation bug affecting StarCoder2-15B results (§7.5), RoPE θ parsing bug for 15B (§6.3), representation bias toward high-resource languages, PII identification challenges, and the risk of malicious code generation with specific CWE examples."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. While it acknowledges some weaknesses (e.g., StarCoder2-7B underperformance), it does not clearly bound the scope of generalization — e.g., these are base model results only, instruction-tuning may change rankings, real-world coding tasks differ from benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The training dataset (The Stack v2) is released on Hugging Face with SWHIDs for traceability. The 'Am I in The Stack' tool and Data Portraits Bloom filter enable membership checking. Evaluation benchmarks are all public."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described in extensive detail in §2: Software Heritage archive extraction, GHArchive for issues and PRs, Kaggle API, package manager crawling, textbook scraping, with specific versions and dates (SH graph dataset 2023-09-06)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper uses public source code repositories and standard benchmarks. The 15 BigCode community annotators for data inspection (§2.1) are mentioned but this is part of data curation, not an evaluation study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: extraction → license detection → language detection → visual inspection → basic filters → language-specific filters → deduplication → PII redaction → decontamination → malware removal → opt-out handling. Table 3 summarizes which steps apply to each data source. Specific removal counts are given (e.g., 59,442 malware files, 22,066 opt-out files)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "§11 (Acknowledgements) discloses funding: NSF awards SES-2326173, SES-2326174, CCF-2131943, Roblox sponsorship, Adams Fellowships, ServiceNow and Hugging Face stewardship, NVIDIA partnership for training infrastructure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hugging Face, ServiceNow Research, NVIDIA, and numerous academic institutions. The corresponding authors are from Hugging Face and ServiceNow."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Hugging Face and ServiceNow, which steward the project and employ many authors, have direct financial interest in StarCoder2's success. ServiceNow's Q4 earnings report and 52% productivity claim are cited in the paper (Yahoo Finance, 2024). NVIDIA provided compute and employs co-authors."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. Multiple authors work at companies (Hugging Face, ServiceNow, NVIDIA, Salesforce, IBM Research, Roblox, etc.) with commercial interests in code generation tools, but no formal declaration of financial interests or conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The SH graph dataset version is 2023-09-06 (§2.1). GHArchive data, StackOverflow dump (2023-09-14), and other sources have specified collection dates. The opt-out cutoff was November 20, 2023."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§3.3 describes decontamination: removing files containing docstrings/solutions from HumanEval, MBPP, APPS, GSM8K questions, and DS1000 prompts, with whitespace-insensitive matching. RepoBench v1.1 removes duplicates against The Stack v2."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly decontaminates against test benchmarks (§3.3), and RepoBench v1.1 was specifically constructed to avoid data leakage by sourcing from repositories created after October 6, 2023. However, not all benchmarks are decontaminated (docs, LHQ, arXiv, Wikipedia excluded)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a model training and benchmark evaluation paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported for any benchmark evaluation. The number of samples generated per benchmark is stated but not the computational cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "§6.4 reports CO2 emissions and GPU hours: 3B model used 97,120 GPU hours on A100s (16,107 kgCO2eq total), 7B model used 145,152 GPU hours on H100s (29,623 kgCO2eq). 15B estimates were pending. FLOPs are reported in Table 6."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. Results appear to be from single training runs. No variance across seeds is reported for any benchmark evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of samples per benchmark evaluation is stated: 50 for MultiPL-E, 40 for DS-1000, 20 for HumanEvalFix, 100 for CanItEdit, 10 for CRUXEval, 25 for security benchmark, 1 for greedy decoding on HumanEval/MBPP."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper mentions 'preliminary experiments' and 'preliminary ablation study' for architecture choices (RoPE, FIM variants) but does not quantify the search process."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Architecture and training decisions (RoPE, GQA, FIM rate, data composition) are stated as chosen but the selection process is not described. The paper mentions 'we explored several FIM variants in preliminary experiments' without detailing results or justifying the final selection."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied despite comparing many models across many benchmarks."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own models against baselines but do not acknowledge self-comparison bias. They use standard benchmarks and evaluation frameworks, which partially mitigates this, but the bias is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "FLOPs are reported in Table 6 but performance is not analyzed as a function of compute budget. StarCoder2-15B uses more training tokens (4.1T) than DeepSeekCoder-33B's training budget, but this compute imbalance is not discussed when comparing performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses EvalPlus as addressing construct validity issues in HumanEval and MBPP (§7.1.1, Listings 1-2): insufficient tests, wrong test cases, and ambiguous descriptions. This demonstrates awareness that benchmarks may not measure what they claim."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are evaluated directly on benchmarks without agent frameworks. The Issue prompt comparison is a prompt format variation, not a scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "RepoBench v1.1 specifically uses repositories created after October 2023 to avoid temporal leakage. The decontamination step (§3.3) removes known benchmark content. Training data dates are specified."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation setups provide hints not available in real usage (e.g., whether docstrings in HumanEval give excessive information compared to real-world coding tasks)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The deduplication removes near-duplicates within training data but there is no discussion of structural similarities between training data and benchmark problems (e.g., whether competitive programming solutions in training overlap with HumanEval-style problems)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Concrete decontamination method applied (§3.3): string matching (with whitespace removal for improved recall) against HumanEval docstrings/solutions, MBPP, APPS, GSM8K, and DS1000. RepoBench removes duplicates against The Stack v2."
       }
     }
   },
   "claims": [
     {
       "claim": "StarCoder2-3B outperforms other Code LLMs of similar size on most benchmarks and matches or surpasses StarCoderBase-15B.",
       "evidence": "Tables 9-10 show StarCoder2-3B leading on HumanEval (31.7), HumanEval+ (27.4), MBPP (57.4), MBPP+ (47.4) among small models, and outperforming StarCoderBase-15B on 11/18 MultiPL-E languages.",
       "supported": "strong"
     },
     {
       "claim": "StarCoder2-15B significantly outperforms other models of comparable size and matches or outperforms CodeLlama-34B.",
       "evidence": "Table 9: StarCoder2-15B scores 46.3 on HumanEval vs CodeLlama-13B's 37.8. Tables 10, 14: outperforms CodeLlama-34B on 10/18 MultiPL-E languages and GSM8K (65.1 vs 54.2).",
       "supported": "strong"
     },
     {
       "claim": "StarCoder2-15B outperforms DeepSeekCoder-33B on math and code reasoning benchmarks.",
       "evidence": "Table 14: GSM8K 65.1 vs 58.7. Table 15: CRUXEval-I Pass@1 48.1 vs 46.5. However, DeepSeekCoder-33B leads on CRUXEval-O (48.6 vs 47.1).",
       "supported": "moderate"
     },
     {
       "claim": "The Stack v2 is 4× larger than The Stack v1 training dataset.",
       "evidence": "Table 4 shows 913B+ unique tokens for the 15B model vs ~232B for StarCoderBase (referenced in text). Table 1 compares v1 and v2 sizes across languages.",
       "supported": "strong"
     },
     {
       "claim": "Pre-training on pull requests (StarCoder2 Issue format) is a viable alternative to pre-training on commits (StarCoderBase Commit format).",
       "evidence": "Table 12: StarCoder2-15B with Issue prompt (38.7 avg) vs StarCoderBase-15B with Commit prompt (26.7 avg) on HumanEvalFix. Table 13: Similar advantage on CanItEdit.",
       "supported": "weak"
     },
     {
       "claim": "StarCoder2-7B underperforms relative to expectations — it does not perform as well as StarCoder2-3B and StarCoder2-15B for their size.",
       "evidence": "Tables 9-10: DeepSeekCoder-6.7B significantly outperforms StarCoder2-7B (47.6 vs 35.4 on HumanEval). The authors explicitly acknowledge this and state they do not know why.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "StarCoder2 introduces a family of open-weight code LLMs (3B/7B/15B) trained on The Stack v2, a 4× larger dataset built on Software Heritage's archive spanning 619 languages. StarCoder2-3B is state-of-the-art for its size class, and StarCoder2-15B matches or outperforms CodeLlama-34B (a model 2× its size) on many benchmarks while excelling at math reasoning (GSM8K: 65.1 vs 54.2) and low-resource languages. The project demonstrates a fully transparent training pipeline with released data, SWHIDs, opt-out mechanisms, and decontamination, setting a standard for open Code LLM development. Notable weaknesses include StarCoder2-7B's underperformance and a FIM implementation bug degrading StarCoder2-15B's infilling capability.",
   "red_flags": [
     {
       "flag": "Non-independent funders",
       "detail": "Hugging Face and ServiceNow steward the project, employ many authors, and have commercial interests in StarCoder2's success. ServiceNow's 52% productivity increase claim (Yahoo Finance, 2024) from StarCoder-based products is cited in the paper's introduction. No competing interests statement is present."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "The paper uses 'significantly outperforms' and 'outperforms' throughout but never performs statistical significance tests. All comparisons are point estimate differences. With sampling-based evaluation, differences could be within noise margins."
     },
     {
       "flag": "Confounded causal claims about training choices",
       "detail": "Claims about the benefits of repository-context training, pull request data, or data composition changes cannot be attributed to individual factors because many variables changed simultaneously (architecture, data size, data composition, tokenizer, training length)."
     },
     {
       "flag": "Missing 15B CO2 estimate",
       "detail": "The paper states 'The paper will soon be updated with estimates for the 15B model' for CO2 emissions — this was not completed, leaving the largest model's environmental cost undocumented."
     },
     {
       "flag": "High insecure code generation rate",
       "detail": "StarCoder2-7B (39.8%) and StarCoder2-15B (39.2%) have the highest insecure code rates among similar-sized models on the security benchmark, higher than all baselines. The paper downplays this as a 'side-effect' of generating more correct code."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval and Codex, foundational benchmarks and models for code generation evaluation."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Predecessor model and dataset (The Stack v1, StarCoder) that this work directly builds upon."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major baseline model family for code generation comparison."
     },
     {
       "title": "DeepSeek-Coder: when the large language model meets programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Primary competitor showing strongest performance on code completion benchmarks."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduced EvalPlus framework addressing inadequate tests in HumanEval/MBPP, adopted for evaluation."
     },
     {
       "title": "MultiPL-E: a scalable and polyglot approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "doi": "10.1109/TSE.2023.3267446",
       "relevance": "Multi-language code generation benchmark used for cross-language evaluation."
     },
     {
       "title": "OctoPack: instruction tuning code large language models",
       "authors": ["Niklas Muennighoff", "Qian Liu", "Armel Randy Zebaze"],
       "year": 2024,
       "relevance": "HumanEvalFix benchmark for code fixing and instruction-tuned model evaluation."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan"],
       "year": 2022,
       "relevance": "Security vulnerability benchmark for evaluating code generation safety."
     },
     {
       "title": "The impact of AI on developer productivity: Evidence from GitHub Copilot",
       "authors": ["Sida Peng", "Eirini Kalliamvakou", "Peter Cihon", "Mert Demirer"],
       "year": 2023,
       "arxiv_id": "2302.06590",
       "relevance": "Key study on AI coding tool productivity impact, cited for Copilot's 56% productivity increase claim."
     },
     {
       "title": "Scaling data-constrained language models",
       "authors": ["Niklas Muennighoff", "Alexander M Rush", "Boaz Barak"],
       "year": 2023,
       "relevance": "Foundational work on data repetition and multi-epoch training for LLMs, directly informing StarCoder2's training strategy."
     },
     {
       "title": "CRUXEval: a benchmark for code reasoning, understanding and execution",
       "authors": ["Alex Gu", "Baptiste Rozière", "Hugh Leather"],
       "year": 2024,
       "arxiv_id": "2401.03065",
       "relevance": "Code reasoning benchmark where StarCoder2-15B outperforms much larger models."
     },
     {
       "title": "The gradient of generative AI release: Methods and considerations",
       "authors": ["Irene Solaiman"],
       "year": 2023,
       "arxiv_id": "2302.04844",
       "relevance": "Framework for understanding openness in LLM development, directly informing BigCode's release strategy."
     }
   ]
 }