ai-research-survey

scan.json (32413B)

---

 {
   "paper": {
     "title": "MultiPL-E: A Scalable and Extensible Approach to Benchmarking Neural Code Generation",
     "authors": [
       "Federico Cassano",
       "John Gouwar",
       "Daniel Nguyen",
       "Sydney Nguyen",
       "Luna Phipps-Costin",
       "Donald Pinckney",
       "Ming-Ho Yee",
       "Yangtian Zi",
       "Carolyn Jane Anderson",
       "Molly Q Feldman",
       "Arjun Guha",
       "Michael Greenberg",
       "Abhinav Jangda"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2208.08227",
     "doi": "10.1109/TSE.2023.3267446"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MultiPL-E translates Python code generation benchmarks (HumanEval, MBPP) to 18 additional programming languages using lightweight compilers. Codex matches or exceeds Python performance on JavaScript, C++, TypeScript, and Scala. Performance correlates with language popularity, but some niche languages (Lua, Julia) perform surprisingly well. Perplexity does not correlate with code correctness, and static type annotations neither help nor hinder performance overall.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The MultiPL-E system, dataset, and tutorial are available at github.com/nuprl/MultiPL-E' (§1, §9). A working repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The translated benchmark datasets are released at the same GitHub repository. The underlying HumanEval and MBPP benchmarks are also publicly available. Appendix B confirms 'The dataset is publicly available at https://github.com/nuprl/MultiPL-E.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes a containerized sandbox for evaluation (§1) and specifies exact language versions for all 18 target languages in Appendix A (e.g., Python 3.10, C++17 with g++17, Java OpenJDK 17, Rust 1.59.0, etc.). The repository includes the containerized evaluation framework."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A states 'Technical information regarding running experiments and evaluating generated programs can be found at github.com/nuprl/MultiPL-E.' The paper also includes a tutorial reference. Specific experimental parameters (temperature, top-p, sample count) are stated in §4.2."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses binomial mixed-effects models (Appendix C) that report coefficient estimates with standard errors (e.g., 'bβ = -2.59 (+/- 0.3)'). These standard errors provide uncertainty quantification for all comparisons."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Extensive mixed-effects models with p-values are reported throughout §5-6 and Appendix C. For example, differences between language frequency categories are tested with specific p-values (§5.1.3: 'p = 0.006; p < 0.001; p = 0.002'). The lme4 library in R is used for fitting (Appendix C)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Pass@1 rates are reported as absolute percentages with differences. For example, §5.1.2: 'Codex's performance on JavaScript is better than its performance on Python... (+2.3%; p = 0.43).' Ablation results report percentage differences (§6.1: 'translating natural language terminology has a small but reliable effect'). Absolute rates provide baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper justifies the 200-sample size: 'we take 200 completions at each temperature and calculate average pass rate using the unbiased sampling estimator' (§4.2). Footnote 6 adds: 'pass@1 rates appear to stabilize around 20 samples, suggesting that future work could achieve a stable estimate with a less computationally costly sample size.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "The unbiased sampling estimator from Chen et al. [1] computes expected pass rates from 200 stochastic completions per problem, inherently accounting for sampling variance. The mixed-effects models in Appendix C report standard errors for all coefficients and treat problem number as a random effect to account for per-problem variability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three models are compared against each other (Codex, CodeGen, InCoder) and against previously reported results. The paper replicates Python results from prior work (§5.1.1, §5.2.2) as a validation baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Codex (codex-davinci-002), CodeGen (16.1B), and InCoder (6.7B) were state-of-the-art code generation models at the time of evaluation (2022). These represent the best available models from three different research groups."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "§6.1 presents a systematic ablation study of MultiPL-E's translation components with four conditions: Original Prompt, Test-Only Translation, Full Translation, and No Doctests. Additional ablations in §6.2 (type annotations), §6.3 (comment style, argument naming), and §6.4 (language features) isolate individual factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports pass@1, pass@10, and pass@100 (§4.2). It also examines perplexity (§5.1.4) and provides fine-grained error categorization (§6.5, Appendix D)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 and Appendix D present a manual error analysis by language experts for four languages (Python, C#, Swift, Racket). 'A language expert then performed a manual investigation of a subset of the completions to derive a set of common error types' (Appendix D.1). This constitutes human evaluation of model outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Unit tests are explicitly hidden from the model. For MBPP, the paper modifies the original format: 'We therefore remove the assertions from the MBPP prompts so that we can use them as hidden unit tests' (§3.1). HumanEval also uses hidden tests by design."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by language (Fig 6), language frequency category (Fig 7, 9), data type features (Fig 11), translation component (Fig 10), and individual error categories (Tables 25-28). Statistical models are also fitted per-language (Appendix C)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§6.5 and Appendix D provide extensive error analysis across four languages with specific error categories (RUNTIME, STATIC, TYPE, LANGUAGE, MODEL), concrete code examples of failures (Figures 17-28), and counts of each error type."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: perplexity does not correlate with code correctness (§5.1.4); type annotations have no overall effect on performance (§6.2, p=0.33); Codex generates Markdown instead of Racket code (§6.5); Python type annotations don't improve Python performance (§6.2.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: multi-language evaluation of three models (§5), Codex matching/exceeding Python performance for several languages (§5.1.2, Fig 6), exploration of language frequency and features (§5.1.3, §6.4), and scalability of the approach (18 languages with ~200 LOC compilers, Table 1)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims from ablation studies (e.g., 'translating doctests and Python-specific terminology has little impact on better-performing languages' §6.1) are justified by controlled single-variable manipulation. Correlational findings (language frequency vs performance) are appropriately described as correlations, not causes."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its generalizations: §7 notes 'the (translated) benchmarks may not be representative of the kinds of problems that programmers typically solve in each language' and 'performance on benchmarks may not accurately represent real-world performance.' Claims are tied to specific models, benchmarks, and languages tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§7 discusses multiple alternative explanations: prompts may not be optimally designed for each language, sampling configurations may not be optimal for non-Python, Codex training set is not public so contamination cannot be ruled out, and benchmark tasks may not be representative of real-world programming."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between what it measures (pass@k on translated benchmarks) and the broader goal (multi-language code generation capability). §7 explicitly warns that benchmark performance may not represent real-world performance. The paper frames its results in terms of benchmark pass rates, not broad capability claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'codex-davinci-002' (§4.1, §5.1.1), InCoder 6.7B parameter model (§4.1), and 'CodeGen 16.1B parameter' multilingual model (§4.1). These are specific enough to identify the exact models used."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Example prompts are shown in Figures 1, 3, 4, 5. Full translation details with stop tokens and language-specific decisions are documented in Appendix A. The complete prompt generation system is released as open-source code at the repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§4.2 states: 'We calculate pass@1 with temperature 0.2, and use temperature 0.8 for pass@10 and pass@100.' Top-p sampling is discussed in §2.2. Stop sequences are detailed per-language in Appendix A."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are queried directly with prompts and generate completions — no tools, feedback loops, or multi-step workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§3 documents the entire compilation pipeline: how unit tests are translated (§3.3.1), type translation and inference (§3.3.2), doctest translation (§3.3.3), terminology translation (§3.3.4). §3.4 documents what was excluded (3 HumanEval benchmarks) and modified (2 with randomized tests). MBPP modifications are described in §3.1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§7 'Threats to Validity' provides substantive discussion of three categories of threats: benchmark representativeness, prompt design sensitivity, and sampling configuration."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§7 discusses threats specific to this study: 'we evaluate both scripting languages and systems languages on the same task, but programmers frequently use these languages for very different tasks'; prompt sensitivity could improve individual language results; sampling settings optimized for Python may not be optimal for others."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§7 explicitly states scope limitations: benchmarks are 'a mix of basic programming problems and straightforward interview questions' that 'may not accurately represent real-world performance.' §3.4 states which languages cannot be supported (C, SQL) and why. Footnote 10 acknowledges sampler configuration exploration is 'beyond the scope of an academic group.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset, benchmark translations, and evaluation framework are released at github.com/nuprl/MultiPL-E. Appendix B confirms: 'The dataset is publicly available' and 'The original data for both datasets is publicly available.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data comes from two existing benchmarks (HumanEval, MBPP) whose provenance is described. §3 documents how translations were produced via the compilation pipeline. Appendix B provides a full datasheet following Gebru et al. [44]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks (HumanEval, MBPP) that are publicly available."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from Python benchmarks to translated versions is fully documented in §3. §3.4 documents what was excluded (3 HumanEval problems) and modified (2 problems). Appendix B states the timeframe (May-October 2022) and that preprocessing was manual with original data preserved."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "§10 states: 'This work was partially supported by the National Science Foundation grant CCF-2052696.' GPU loans from Steven Holtzen and Joydeep Biswas and support from Northeastern Research Computing are acknowledged."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are listed with a note that they are alphabetically ordered with students first, then faculty. The authors are academic researchers, not affiliated with the companies whose models they evaluate."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Science Foundation is the funder and has no financial interest in the comparative performance of Codex, CodeGen, or InCoder. This is independent academic funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While the authors appear to be academic researchers with no obvious conflicts, absence of a declaration is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoff dates are not stated for any of the three models. For Codex: 'Details of its training set and size are not public' (§4.1). InCoder and CodeGen training data is described in terms of sources but no cutoff dates are given."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§5.1.2 explicitly discusses train/test overlap: 'The Codex training set is not public; it is possible that the latest model has been trained on solutions to the HumanEval benchmarks in Python, and this could be inflating its performance.' They argue the translated versions are new datasets, mitigating this concern."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§5.1.2 addresses contamination: 'MultiPL-HumanEval is a new dataset for the 18 other languages. That Codex matches or exceeds its Python performance on some of these new languages suggests a negligible impact of any train/test overlap.' The novelty of the translated benchmarks is presented as a natural contamination mitigation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using code generation models and unit tests."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Appendix B confirms: 'Not applicable. The dataset adapts an open source dataset released under the terms of the MIT license.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or wall-clock times are reported despite generating 200 completions per problem across 19 languages for 3 models (over 2 million completions total). Footnote 10 acknowledges resource intensity but does not quantify it."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total compute budget is stated. The paper thanks individuals for GPU loans and Northeastern Research Computing for support (§10), but does not quantify GPU hours, API costs, or total computation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While 200 stochastic completions per problem are sampled, the paper does not report variation across independent runs or random seeds. The unbiased estimator provides a point estimate but no seed sensitivity analysis is presented."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§4.2 clearly states: 'we take 200 completions at each temperature.' Two temperature settings are used (0.2 for pass@1, 0.8 for pass@10 and pass@100)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature values (0.2, 0.8) and top-p sampling are adopted from prior work [1] without reporting any search budget. Footnote 10 acknowledges that 'The original experiment on sampler configurations by Chen et al. [1] has not been repeated by any lab' but does not attempt their own search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The sampling configuration follows established practice from Chen et al. [1]: 'We use the same sampling configuration that is used in most prior work on code generation' (§7). The paper explicitly states temperature 0.2 for pass@1 (§4.2) rather than selecting the best of multiple configurations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Appendix C reports dozens of statistical comparisons across languages, models, and experimental conditions (Tables 2-24) without applying Bonferroni, Holm, or any other multiple comparison correction. Many comparisons are reported as significant at p < 0.05."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own benchmark translation system but do not acknowledge potential bias from evaluating their own work. They compare against MBXP (§8) and note differences in methodology, but do not discuss how authorship of MultiPL-E might influence their evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The three models differ substantially in size (InCoder 6.7B, CodeGen 16.1B, Codex unknown) but performance is not analyzed as a function of compute budget or model size. The compute disparity is not controlled for in comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§7 explicitly discusses construct validity: 'the (translated) benchmarks may not be representative of the kinds of problems that programmers typically solve in each language. For example, we evaluate both scripting languages and systems languages on the same task, but programmers frequently use these languages for very different tasks.' §5.2.1 also compares HumanEval and MBPP difficulty."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are queried directly with prompts and produce completions without any agentic framework or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§5.1.2 discusses temporal leakage: the original HumanEval Python solutions may be in Codex's training data since HumanEval was published before the model was trained. The paper argues the translated 18-language versions are new and thus not contaminated."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses feature leakage in MBPP: the original benchmark 'prompts the model with the same unit tests it uses to test the generated code' (§3.1). MultiPL-E fixes this by hiding the test cases. They also critique MBXP for this same leakage (§8)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether translated problems are statistically independent from training data beyond the Python contamination discussion. No analysis of structural similarity between benchmark problems and training code is provided."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap) is applied. The paper argues that translated benchmarks are new datasets as a mitigation strategy, but does not empirically verify the absence of leakage."
       }
     }
   },
   "claims": [
     {
       "claim": "Codex matches or exceeds Python performance on JavaScript, C++, Scala, and TypeScript on MultiPL-HumanEval",
       "evidence": "§5.1: Mixed-effects model finds no reliable differences between Python and these 4 languages (p=0.43 for JavaScript, p=0.77 for C++, p=0.10 for Scala, p=0.39 for TypeScript). Figure 6 shows pass@1 rates. JavaScript actually exceeds Python by 2.3%.",
       "supported": "strong"
     },
     {
       "claim": "There is no strong correlation between model perplexity and correctness of generated code",
       "evidence": "§5.1.4, Figure 8: Perplexity is highest for JavaScript and TypeScript while Codex performs best on these. However, the comparison uses perplexity scores from an older Codex model reported by Xu et al. [7].",
       "supported": "moderate"
     },
     {
       "claim": "Code generation performance correlates with language popularity, with some niche language exceptions",
       "evidence": "§5.1.3: Mixed-effects model finds reliable differences between HIGH and other frequency categories (MEDIUM p=0.006, LOW p<0.001, NICHE p=0.002). However, Lua (NICHE) is 9th-best and Scala (LOW) performs well.",
       "supported": "strong"
     },
     {
       "claim": "Static type-checking neither helps nor hinders code generation performance",
       "evidence": "§6.2: Mixed-effects model finds no significant effect of type annotations on Codex pass@1 for HumanEval (p=0.33) or MBPP (p=0.23). Removing Python type annotations has no effect (p=0.23). However, precise types improve TypeScript performance vs Any types (-2.5%, p<0.001).",
       "supported": "strong"
     },
     {
       "claim": "Code generation performance is sensitive to prompt design choices for both niche and popular languages",
       "evidence": "§6.1 ablation: Significant differences between Full Translation and Test-Only (p=0.03) and between No Doctests and Test-Only (p<0.001). §6.3: Multi-line comments improve Racket (+1.9%, p<0.001) but hurt PHP (-3.1%, p=0.001). Argument naming improves Perl by 8% (p<0.001).",
       "supported": "strong"
     },
     {
       "claim": "MBPP is a less challenging benchmark than HumanEval",
       "evidence": "§5.2.1: MultiPL-MBPP pass@1 is higher than MultiPL-HumanEval for all but 6 of 57 model/language pairs, despite MBPP lacking doctests which are shown to help performance.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Codex training data opacity",
       "detail": "Codex's training data and model size are not public (§4.1). The paper cannot rule out that codex-davinci-002 was trained on HumanEval solutions or the translated benchmark problems. The contamination analysis relies on the argument that translations are new, but cannot verify this empirically."
     },
     {
       "flag": "No compute costs reported",
       "detail": "The evaluation generates over 2 million completions (200 samples × ~560 problems × 19 languages × 3 models × 2 temperatures) but reports no API costs, GPU hours, or wall-clock time. This makes cost-benefit assessment impossible and hinders reproducibility for resource-constrained researchers."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Appendix C contains dozens of statistical tests across languages, models, and experimental conditions without any family-wise error rate correction. With ~20 language comparisons per model, some significant results could be spurious."
     },
     {
       "flag": "Model size confound in cross-model comparisons",
       "detail": "InCoder (6.7B), CodeGen (16.1B), and Codex (unknown size) are compared directly without controlling for model size or compute. Performance differences could partly reflect scale rather than training approach."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex model, the primary benchmark and one of three models evaluated in this paper."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["J. Austin", "A. Odena", "M. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark, one of the two benchmarks translated by MultiPL-E."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp", "B. Pang", "H. Hayashi"],
       "year": 2022,
       "arxiv_id": "2203.13474",
       "relevance": "Introduces CodeGen, one of the three code generation models evaluated across 19 languages."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["D. Fried", "A. Aghajanyan", "J. Lin"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Introduces InCoder, one of the three code generation models evaluated. Demonstrates code infilling capability."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["F. F. Xu", "U. Alon", "G. Neubig", "V. J. Hellendoorn"],
       "year": 2022,
       "relevance": "Evaluates code LLMs on 12 languages using perplexity (not unit tests), providing the perplexity scores used in MultiPL-E's correlation analysis."
     },
     {
       "title": "Productivity assessment of neural code completion",
       "authors": ["A. Ziegler", "E. Kalliamvakou", "X. A. Li"],
       "year": 2022,
       "relevance": "Evaluates developer productivity with neural code completion tools, relevant to understanding real-world code generation impact."
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "authors": ["B. Athiwaratkun", "S. K. Gouda", "Z. Wang"],
       "year": 2022,
       "arxiv_id": "2210.14868",
       "relevance": "MBXP: concurrent multi-language benchmark effort. MultiPL-E critiques its methodology (visible test cases, greedy decoding, less faithful type translation)."
     },
     {
       "title": "CodeT: Code generation with generated tests",
       "authors": ["B. Chen", "F. Zhang", "A. Nguyen"],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Uses generated tests to improve code generation, reports codex-davinci-002 results used for Python replication comparison."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Y. Li", "D. Choi", "J. Chung"],
       "year": 2022,
       "arxiv_id": "2203.07814",
       "relevance": "Demonstrates code generation on competitive programming problems, a related but distinct evaluation paradigm."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["D. Hendrycks", "S. Basart", "S. Kadavath"],
       "year": 2021,
       "arxiv_id": "2105.09938",
       "relevance": "APPS benchmark for code generation from competitive programming, identified as a potential future translation target for MultiPL-E."
     },
     {
       "title": "GPT-NeoX-20B: An open-source autoregressive language model",
       "authors": ["S. Black", "S. Biderman", "E. Hallahan"],
       "year": 2022,
       "arxiv_id": "2204.06745",
       "relevance": "Open-source large language model used for code generation tasks, relevant to the code generation model landscape."
     },
     {
       "title": "PaLM: Scaling language modeling with Pathways",
       "authors": ["A. Chowdhery", "S. Narang", "J. Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "Large language model with code generation capabilities (PaLM-Coder), noted as unavailable for academic evaluation."
     }
   ]
 }