ai-research-survey

scan.json (29667B)

---

 {
   "paper": {
     "title": "A Performance Study of LLM-Generated Code on Leetcode",
     "authors": [
       "Tristan Coignion",
       "Clément Quinton",
       "Romain Rouvoy"
     ],
     "year": 2024,
     "venue": "International Conference on Evaluation and Assessment in Software Engineering (EASE 2024)",
     "arxiv_id": "2407.21579",
     "doi": "10.1145/3661167.3661221"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Across 18 code-generation LLMs evaluated on 204 Leetcode problems, runtime performance of generated code is largely similar regardless of model (Cohen's d = 0.024). LLM-generated code is on average faster than 73% of human submissions on Leetcode. Temperature increases performance variance but not mean performance (correlation 0.41 with variance, 0.05 with mean). The study also reveals serious data contamination issues in Leetcode evaluations, with a tenfold decrease in pass@k when using post-training-cutoff problems.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states: 'All the artifacts of this study, including our results, code, and datasets, are available in the following public repository: https://zenodo.org/doi/10.5281/zenodo.7898304.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package includes results and datasets (Section 3.4). Leetcode problems are publicly available, and the canonical solutions are from public sources (WalkCC repository, Leetcode community)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (AMD EPYC 7301, 16 cores, 128GB memory, Grid5000 chiclet cluster in Section 3.3) and tools are mentioned (Deepspeed, pytest-benchmark), but no library versions, requirements.txt, or detailed software environment specification is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 provides a Zenodo replication package with code and datasets. A companion notebook is also mentioned for results exploration. The methodology sections (2-4) describe the full procedure in reproducible detail."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows '95% confidence interval' for pass@1 results across difficulty levels and datasets."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes pairwise Student t-tests on mean run times to determine significant differences between LLMs."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 reports Cohen's d: 'the mean Cohen's d effect size measures a mere 0.024.' This contextualizes the statistical significance of performance differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the choice of 204 problems, 10 solutions per problem per temperature, or the minimum of 5 valid solutions for pairwise comparison. The sample sizes are stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation of canonical solution run times is reported across runs (Section 3.3: 'over 96% had a standard deviation lower than 1/10th of their average run time'). Coefficients of variation are reported in Figure 3 (0.089 Leetcode vs 0.035 local)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-written canonical solutions serve as baselines for performance comparison. The 18 LLMs are also compared against each other via pairwise analysis."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The 18 LLMs evaluated (CodeLlama, StarCoder, WizardCoder, CodeGen2.5, etc.) were all contemporary state-of-the-art code models at the time of the study (March-September 2023). Table 1 lists them."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates existing LLMs on code generation tasks rather than proposing a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: pass@1, pass@10, local run time (via pytest-benchmark), Leetcode time rank, memory usage, coefficient of variation, Cohen's d, and correlation coefficients."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: Leetcode judge for correctness, pytest-benchmark for runtime. No human evaluation of code quality, readability, or other qualitative aspects."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 'new' dataset uses only problems published after January 1, 2023, explicitly created as a temporal split to avoid contamination from training data. This serves as a held-out test set by design."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by difficulty level (easy, medium, hard) in Figure 2, by model in Tables 1-2 and Figure 6, and by temperature in the RQ3 analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 details failure modes: 4,863 timeouts, 20 recursion errors, 47 other errors. Section 6.1 discusses InCoder performing worse than expected. Section 5.1.2 identifies Leetcode measurement unreliability. Only 3.6% of generated solutions were valid."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings: Leetcode measurements are unreliable (RQ1), memory usage measures decrease over time, rankings are affected by submissions, the low 3.6% solution validity rate, and performance differences between LLMs are negligibly small despite statistical significance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: comparable performance across LLMs (Cohen's d = 0.024, Section 5.2), code more efficient than human average (73% rank, Section 5.4), data contamination discussion (RQ1, Section 5.1.1). The abstract's hedging ('on average') matches the evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main quasi-causal claim (temperature affects performance variance) is supported by controlled experimental variation of temperature as an independent variable across the same models and problems. Language is appropriately correlational ('moderately correlated (0.41)')."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'LLMs produce code with comparable performance, irrespective of the adopted LLM' but only 18 models were tested, all code-specific open-source models (0.35B-15.5B parameters) plus Copilot. No GPT-4, Claude, or larger models. Only Python, only algorithmic problems. Section 6.2 discusses some bounds but the abstract/conclusion claims are broader than the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 considers that 'The observed shift in functional validity between the two datasets could also arise from a genuine difference in the difficulty of the questions.' Section 6.1 discusses shared training datasets as an explanation for similar performance and notes InCoder may perform worse due to using it for left-to-right generation instead of infilling."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (runtime via pytest-benchmark) and its claims are specifically about runtime performance on algorithmic problems. It does not conflate runtime with broader 'code quality' — Section 6.2 explicitly acknowledges they do not measure memory usage and results are limited to algorithmic problems."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists specific model names with sizes (e.g., 'CodeGen-Mono 6B', 'CodeLlama-7B-instruct', 'StarCoder 15.5B') that map to specific Hugging Face model identifiers. GitHub Copilot is acknowledged as closed-source without version control."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows a complete example prompt with the actual Leetcode problem description and function signature. Section 2.2 ('Input prompts') describes exactly what is included (problem description only, no examples or constraints) and why."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports: temperatures (0.1, 0.2, 0.4, 0.6, 0.8, 1.0), nucleus sampling top_p = 0.95, max tokens = 600. Copilot's default temperature is noted as non-configurable."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The study performs direct code generation from prompts to LLMs with no tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the 3-step validation pipeline with exact counts: 210,120 generated → local validation → Leetcode validation → timeout/error exclusion → 7,481 valid solutions (3.6%). Error breakdown: 4,863 timeouts (98.6%), 20 recursion errors (0.04%), 47 other errors (0.1%)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 'Limits and Threats to Validity' is a dedicated subsection with substantive discussion of multiple specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 identifies specific threats: Copilot retraining risk, reliance on Leetcode's test suite potentially favoring some implementations, 10-second timeout cut-off, missing memory analysis, recursion limit differences between local and Leetcode environments, and that algorithmic problems don't generalize to all programming domains."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.2 explicitly states what was NOT tested: memory usage ('we do not consider memory usage at all'), non-algorithmic code ('only evaluate the LLMs on algorithmic problems, the performances of the LLMs are hard to generalize across all programming fields'), and notes the ranking comparison limitations ('the ranking evolves and we have no information about the population')."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4: 'All the artifacts of this study, including our results, code, and datasets, are available in the following public repository: https://zenodo.org/doi/10.5281/zenodo.7898304.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes problem selection (204 problems published after Jan 1, 2023), how test cases were extracted (crawling Leetcode instructions, submitting modified solutions to trigger timeout failures), and how canonical solutions were sourced (WalkCC repository, Leetcode community upvoted solutions)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public platforms (Leetcode problems, Hugging Face models) and their selection is documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented with counts: 210,120 generated solutions → local validation → Leetcode validation → timeout exclusion → 7,481 valid solutions (Section 3.2). Error categories and counts at the final stage are provided (4,863 timeouts, 20 recursion errors, 47 other errors)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses funding from ANR (France 2030 program) through CARECloud, DISTILLER, and KOALA projects, and use of Grid5000 testbed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are from Univ. Lille/CNRS/Inria, France. Affiliations are clearly stated. They evaluate third-party open-source models and a commercial tool (Copilot) — no conflict with their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from ANR (French national research agency) and Grid5000 (academic infrastructure). Neither has a financial stake in LLM performance outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of a disclosure statement does not equal absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states: 'the majority of their training datasets have a cut-off date between 2021 and 2022.' They also acknowledge that GitHub Copilot's training data cutoff is unknown as a closed-source tool."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensively discussed in RQ1 (Section 5.1.1). They found a tenfold decrease in pass@k between old and new datasets, which they attribute to data contamination. They note that '3sum' has ~4,000 matches on GitHub, likely appearing in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Central contribution of the paper. They mitigate contamination by using only problems published after January 1, 2023 (post-training-cutoff). They compare old vs new datasets to demonstrate contamination effects and discuss reproducibility implications for future studies."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on coding problems."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM-generated code on algorithmic problems."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, per-problem generation time, or inference cost is reported. The total number of solutions generated (210,120) is stated but not the associated cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Grid5000 chiclet cluster, AMD EPYC 7301, GPUs with Deepspeed) but total GPU hours, wall-clock time, or compute costs for the full experiment are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. The 10 generations per problem at different temperatures provide some diversity, but seed sensitivity is not explicitly reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states 10 solutions per problem per LLM. Section 3.3 states benchmarks run 'at least 10 times and for at least 1 second in total.' Six temperatures and 18 models are specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Settings like top_p=0.95 and max_tokens=600 are adopted from Chen et al. without reporting alternatives considered. Temperature is varied systematically but no search budget is reported for other hyperparameters."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'we calculated the pass@k for each temperature when evaluating an LLM's functional correctness and considered the best one as the pass@k for that LLM.' The selection criterion is transparent."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Pairwise t-tests are performed across all pairs of 18 LLMs (153 pairs) without any mention of Bonferroni, Holm, or other corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The authors evaluate third-party LLMs, not their own system. There is no self-comparison bias to address."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 350M to 15.5B parameters but performance is not plotted or discussed as a function of model size or compute budget. The finding that model differences are negligible implicitly addresses this but no explicit analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 is entirely dedicated to evaluating Leetcode's validity as a benchmarking platform. They analyze measurement reliability (coefficient of variation), rank stability, memory measure drift, and data contamination effects. They compare Leetcode measures to local benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive direct prompts and generate code completions without agent loops or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Major focus of the paper. Section 2.2: problems published after January 1, 2023 are used as the 'new' dataset, post-dating the training data cutoff of 2021-2022 for most models. The tenfold pass@k decrease between old and new datasets demonstrates the contamination effect."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the Leetcode prompt format or structure could leak information to the models. The prompts are presented exactly as Leetcode formats them, and the paper does not discuss whether this familiar format provides an unfair advantage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Leetcode problems share structural similarities that could introduce non-independence (e.g., similar problem types, data structures, or solution patterns appearing in both training and test sets)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splitting is used as a concrete prevention method (problems after Jan 1, 2023). Additionally, the comparison between old and new dataset pass@k rates (tenfold decrease) serves as an empirical detection method for contamination. Section 5.1.1 also analyzes GitHub search results for specific prompts (~4,000 matches for '3sum')."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs produce code with comparable runtime performance regardless of the model used (Cohen's d = 0.024).",
       "evidence": "Section 5.2: Pairwise t-test comparisons across 18 LLMs on common problems. Mean Cohen's d effect size of 0.024. Figure 6 shows the pairwise comparison matrix.",
       "supported": "strong"
     },
     {
       "claim": "LLM-generated code is on average faster than 73% of human-submitted Leetcode solutions.",
       "evidence": "Section 5.4: Leetcode ranking for CodeGen-6B-mono shows mean rank of 73%. Figure 8 shows the distribution. However, rankings evolve with submissions (Section 5.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "Temperature increases performance variance but does not significantly affect mean performance.",
       "evidence": "Section 5.3: Correlation of 0.05 between temperature and performance (negligible), but 0.41 between temperature and performance variance (moderate). Figure 7 shows correlation distribution.",
       "supported": "strong"
     },
     {
       "claim": "Data contamination causes approximately tenfold inflation in LLM pass@k scores on Leetcode.",
       "evidence": "Section 5.1.1 and Figure 2: Comparing old dataset (pre-2023 problems, likely in training data) to new dataset (post-2023) shows roughly tenfold decrease in pass@k. However, difficulty differences between datasets cannot be fully ruled out.",
       "supported": "moderate"
     },
     {
       "claim": "Leetcode measurements are less reliable than local benchmarking (coefficient of variation 0.089 vs 0.035).",
       "evidence": "Section 5.1.2 and Figure 3: Coefficient of variation comparison. Figure 4 shows scatter plot where Leetcode cannot distinguish clusters visible in local measurements. Correlation between local and Leetcode times is only 0.28.",
       "supported": "strong"
     },
     {
       "claim": "Success rate of LLMs (functional correctness) does not substantially impact code performance.",
       "evidence": "Section 5.3: Only slight negative correlation (-0.08) between success rate and run time, and close to no correlation (-0.11) between success rate and performance variation.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Very low valid solution rate limits comparison power",
       "detail": "Only 7,481 out of 210,120 generated solutions (3.6%) were valid. Only 24 problems had sufficient valid solutions from 10+ models for pairwise comparison. This severely limits statistical power for RQ2."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Pairwise t-tests across 18 LLMs (153 pairs) are performed without Bonferroni or other corrections, inflating the risk of spurious significant differences."
     },
     {
       "flag": "Human comparison methodology is weak",
       "detail": "The Leetcode ranking used for RQ4 (LLMs vs humans) evolves with submissions. The authors' own submissions changed the ranking: Copilot dropped from rank 77 to 54 after testing all models. The 'population' of human submitters is uncharacterized."
     },
     {
       "flag": "Narrow scope of tested models",
       "detail": "All 18 models are code-specific open-source models (350M-15.5B parameters) plus Copilot. No GPT-4, Claude, or other frontier models are tested, but conclusions are framed broadly ('irrespective of the adopted LLM')."
     },
     {
       "flag": "Contamination alternative explanation not fully ruled out",
       "detail": "The tenfold pass@k decrease between old and new datasets is attributed to data contamination, but the paper acknowledges 'quantifying this last hypothesis [difficulty difference] proves to be challenging.' The two datasets differ in composition (95/105/100 vs 56/104/44 by difficulty)."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric used throughout this study for evaluating LLM code generation capability."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM evaluated in this study; achieved highest pass@1 (0.095) on Leetcode problems."
     },
     {
       "title": "Do Users Write More Insecure Code with AI Assistants?",
       "authors": ["Neil Perry"],
       "year": 2023,
       "doi": "10.1145/3576915.3623157",
       "relevance": "Studies security implications of AI code assistants; relevant to understanding real-world impact of LLM-generated code quality."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce"],
       "year": 2022,
       "doi": "10.1109/SP46214.2022.9833571",
       "relevance": "Evaluates security quality of Copilot code suggestions; complements this study's performance-focused evaluation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark and studies temperature effects on code generation, which this paper builds upon."
     },
     {
       "title": "Piloting Copilot and Codex: Hot Temperature, Cold Prompts, or Black Magic?",
       "authors": ["Jean-Baptiste Döderlein"],
       "year": 2023,
       "doi": "10.2139/ssrn.4496380",
       "relevance": "Closely related prior work using Leetcode to evaluate Copilot/Codex; this paper extends the approach with performance measurement."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu"],
       "year": 2024,
       "relevance": "Investigates correctness of LLM-generated code with rigorous evaluation methodology."
     },
     {
       "title": "Assessing the Quality of GitHub Copilot's Code Generation",
       "authors": ["Burak Yetistiren"],
       "year": 2022,
       "doi": "10.1145/3558489.3559072",
       "relevance": "Evaluates Copilot code quality including success rate measurement, a related evaluation dimension."
     },
     {
       "title": "Stop Uploading Test Data in Plain Text: Practical Strategies for Mitigating Data Contamination by Evaluation Benchmarks",
       "authors": ["Alon Jacovi"],
       "year": 2023,
       "arxiv_id": "2305.10160",
       "relevance": "Addresses data contamination in LLM evaluation, directly relevant to this paper's RQ1 findings on benchmark contamination."
     },
     {
       "title": "Learning Performance-Improving Code Edits",
       "authors": ["Aman Madaan"],
       "year": 2023,
       "relevance": "Fine-tunes LLMs for code performance improvement; complements this study's finding that LLMs already generate reasonably performant code."
     },
     {
       "title": "Large Language Models and Simple, Stupid Bugs",
       "authors": ["Kevin Jesse"],
       "year": 2023,
       "doi": "10.1109/MSR59073.2023.00082",
       "relevance": "Studies prevalence of bugs in LLM-generated code, complementing this paper's focus on code performance."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Introduces CodeLlama family of models, five variants of which are evaluated in this study."
     }
   ]
 }