ai-research-survey

scan.json (30836B)

---

 {
   "paper": {
     "title": "Measuring The Impact Of Programming Language Distribution",
     "authors": [
       "Gabriel Orlanski",
       "Kefan Xiao",
       "Xavier Garcia",
       "Jeffrey Hui",
       "Joshua Howland",
       "Jonathan Malmaud",
       "Jacob Austin",
       "Rishabh Singh",
       "Michele Catasta"
     ],
     "year": 2023,
     "venue": "International Conference on Machine Learning",
     "arxiv_id": "2302.01973",
     "doi": "10.48550/arXiv.2302.01973"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "BabelCode is open-sourced at https://github.com/google-research/babelcode (footnote 1 and abstract). The paper states 'BabelCode is open-sourced, has an extensive test suite, and supports evaluating four benchmarks in 14 languages.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (BC-HumanEval, BC-MBPP, BC-Transcoder, TP3) are released through the BabelCode framework. The underlying benchmarks (HumanEval, MBPP, Transcoder, P3) are publicly available. The curated training corpus is not released, but the evaluation data is."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions T5X and SeqIO frameworks but does not provide environment setup details sufficient to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The BabelCode repo likely contains some instructions, but the paper itself does not include a 'Reproducing Results' section or specific commands to replicate the training and evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4-11 are point estimates (pass@k values) with no confidence intervals or error bars. No uncertainty quantification is provided for any result."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., '48.17% improvement compared to PaLM-Coder 8B') based solely on comparing point estimates. No statistical significance tests are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as relative improvements with baseline context throughout. For example, '12.34% higher pass@k across all tasks' (abstract), '66.48% better pass@k on low-resource languages' (abstract), and detailed per-language relative differences in Tables 12-13."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the number of problems per benchmark (161 for BC-HumanEval, 855 for BC-MBPP, etc.), the number of programming languages (14), or the model sizes chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are from single training runs per configuration. No standard deviations, variance across seeds, or spread measures are reported. The pass@k estimator uses n samples (200 or 50) per problem but this addresses sampling noise in the estimator, not experimental variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Models trained on the natural distribution serve as baselines for the balancing experiments. PaLM-Coder 8B and 62B are included as external baselines (Figure 4, Tables 4-11)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "PaLM-Coder (Chowdhery et al., 2022) was a contemporary state-of-the-art model at the time of writing. The paper also compares against PaLM (non-code-finetuned) as an additional reference."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates the Unimax N parameter (N=1,2,3,4) across three model sizes (1B, 2B, 4B) on four benchmarks. This systematically evaluates the contribution of the data balancing strategy at different duplication limits. Results in Figure 5-6 and Tables 4-13."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: pass@1, pass@25, pass@100, plus qualitative prediction-level metrics including % error, % failed tests, % test cases passed per problem, % timed out (Section 5.3, Tables 16-19)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through execution-based pass@k metrics on test suites. No human evaluation of code quality, readability, or correctness beyond test cases is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported on established benchmarks (HumanEval, MBPP, Transcoder, TP3) that are separate from the training data. No dev/test set confusion — the benchmarks are used only for evaluation, and hyperparameters (T=0.8, top_p=0.95) are fixed rather than tuned on the test data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-language breakdowns are provided in all results tables (Tables 4-15), covering all 14 programming languages individually. Additional breakdowns by high-resource vs. low-resource groups are provided in Tables 12-13 and Figures 5-6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 analyzes prediction-level failure modes: compilation errors, runtime errors, timeouts, and partial test case failures. Figures 7, 9, 10 and Tables 16-19 provide detailed failure analysis for both HR and LR languages."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that balanced training hurts high-resource language performance (15.47%, 14.00%, 9.35% losses for 1B, 2B, 4B). It also reports that LR improvements from balanced data 'do not scale with model size' (Section 1). The 1B model on Unimax for BC-Transcoder Python shows a negative average change of -26.04% (Section 5.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's specific numerical claims (12.34% overall improvement, 66.48% LR improvement, 12.94% HR cost, 30.77% LR translation improvement, 19.58% HR translation cost) are supported by the results in Section 5 and the appendix tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about training distribution affecting model performance. The study design is adequate: models are trained from scratch with controlled variables (same architecture, same vocabulary, same total compute budget) varying only the training distribution. This controlled experiment supports causal claims about the effect of data balancing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested setting. The title refers to 'programming language distribution' which matches the scope. Results are reported per-language, per-model-size, and per-dataset. The abstract specifies exact metrics and percentages rather than broad generalizations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses transfer learning from similar languages as an explanation for scaling behavior. Section 5.3 discusses that NL2Code tasks depend on unique code-docstring pairs while translation does not, explaining differential effects of balancing. The paper also discusses that performance gains may stem from unique data rather than duplication (Section 5.3: 'less than 1% improvement when going from Unimax 1 to Unimax 2')."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on code generation and translation benchmarks and claims pass@k performance. It does not inflate measurements into broader claims about 'code quality' or 'developer productivity.' The claims match the granularity of the measurements."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Full architecture details are provided in Table 2: layers, heads, d_model, training tokens for all model sizes (1B, 2B, 4B). PaLM-Coder specifications are also listed. Architecture follows Chowdhery et al. (2022). Vocabulary size (64k tokens) and training framework (T5X, SeqIO) are specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix E (E.1 for generation, E.2 for translation) with concrete examples showing the exact text sent to the model, including the filled-in fields for specific languages."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: learning rate 0.01, Adafactor optimizer, batch size 256, context window 2048, warmup steps = 10% of total, training steps per size (38K/77K/190K). Section 4.5 reports: T=0.8, top_p=0.95, no top_k, n=200/50 samples."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models generate code directly via sampling with no tool use, retry logic, or multi-step reasoning."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the data pipeline: publicly available web code → license filtering (similar to Kocetkov et al.) → heuristic quality filtering → near-deduplication → language segmentation by GitHub Linguist file extensions. Table 3 provides the final language distribution percentages. The Unimax balancing algorithm is described in Section 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion (Section 7) is brief and forward-looking without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no acknowledgment of potential issues with single-run evaluations, benchmark contamination risk, or the limited model size range."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of scope limitations such as: results are specific to decoder-only models, the 14 chosen languages, or the specific training data source."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The curated training corpus is not released. Only the BabelCode evaluation framework is open-sourced. The raw training data (filtered GitHub code) cannot be independently verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes data collection: 'Our curated source code corpus was obtained by collecting publicly available code data on the web using a custom code data collection system.' License filtering, quality heuristics, and near-deduplication are described. Language segmentation by file extension is detailed in Table 3."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are publicly available code from the web and standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The general pipeline steps are described (collection → license filter → quality filter → deduplication → language segmentation) but intermediate counts at each stage are not provided. The paper does not state how many files were removed at each filtering step — only the final distribution percentages are given in Table 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding statement is provided. The acknowledgments section thanks individuals but does not mention grants, corporate funding, or sponsorship. Author affiliations (Google Labs, Google Brain) imply corporate funding but this is not explicitly disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Google Labs, Google Brain, and NYU Department of Computer Science. The footnote notes 'Work Done While At Google' for the first author."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Most authors are Google employees (Google Labs, Google Brain). Google has a commercial interest in code LLM capabilities and the training strategies investigated. The funder is not independent of the outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state when the GitHub training data was collected or what time period it covers. No training data cutoff date is provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, or Transcoder solutions appeared in the curated GitHub training corpus. Given that training data comes from public GitHub and these benchmarks are widely forked/solved on GitHub, this is a significant omission."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021), MBPP (2021), and Transcoder (2020) were publicly available before the training data was presumably collected. No contamination analysis or decontamination was performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a purely computational benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, latency, or wall-clock time for evaluation is reported despite generating 200 programs per problem for HumanEval and 50 for translation tasks across 14 languages and 15 model configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training tokens are stated (20.2B, 40.4B, 100B for 1B, 2B, 4B models) but no GPU hours, hardware type, wall-clock training time, or total compute budget (e.g., FLOPs) is reported. Fifteen models were trained (5 distributions × 3 sizes) — the total compute footprint is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single training runs per configuration. No multi-seed experiments or seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of generated samples per problem is stated (n=200 for HumanEval, n=50 for translation) for the pass@k estimator, but each model configuration appears to be trained and evaluated once. The number of independent experimental replications is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The learning rate (0.01), optimizer (Adafactor), batch size (256), and sampling parameters (T=0.8, top_p=0.95) appear to be fixed choices without reporting how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All Unimax N configurations (N=1,2,3,4) and the natural distribution are reported for every model size and dataset, rather than cherry-picking the best configuration. Figures 5-6 and Tables 4-13 show all results transparently."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes hundreds of comparisons across 14 languages, 5 distributions, 3 model sizes, and 4 datasets without any multiple comparison correction. No Bonferroni, Holm, or other family-wise error rate corrections are applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their BC models against PaLM-Coder, but both are from Google. There is no acknowledgment that comparing their own models/implementations against baselines may introduce systematic bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 notes that the BC-4B model saw 53B more tokens than PaLM-Coder 62B, but performance is not systematically reported as a function of compute budget. The comparison between a 4B model with 100B tokens vs. PaLM-Coder 62B with 46.8B tokens conflates model size and training compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether pass@k on HumanEval, MBPP, Transcoder, or TP3 actually measures 'programming language capability' or 'code generation ability' in a meaningful sense. No construct validity analysis is provided."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models generate code directly via sampling without any agentic scaffold, tool use, or multi-step reasoning."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021), MBPP (2021), and Transcoder (2020) were published before the training data was presumably collected from GitHub. Solutions to these benchmarks are widely available on GitHub. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.5 notes that translated signatures are provided in prompts and dismisses this as 'trivial to translate,' but does not systematically analyze whether the prompt construction leaks information that would not be available in real usage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the curated GitHub training corpus contains code structurally similar to the benchmark problems. GitHub repositories frequently contain solutions to HumanEval and MBPP problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines are mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "Training on a balanced corpus (Unimax) results in 12.34% higher pass@k across all tasks and languages compared to the natural distribution baseline.",
       "evidence": "Section 5.2 and Tables 4-11 provide pass@k results across 4 benchmarks, 14 languages, and 3 model sizes for natural vs. Unimax distributions.",
       "supported": "strong"
     },
     {
       "claim": "Balanced training achieves 66.48% better pass@k on low-resource languages at the cost of only 12.94% decrease on high-resource languages.",
       "evidence": "Section 5.2, Tables 12-13 show per-language relative changes. The LR/HR tradeoff is documented across all model sizes and datasets.",
       "supported": "strong"
     },
     {
       "claim": "Scaling model size mitigates the high-resource performance loss from balanced training, going from 39.70% loss with 1B to 2.47% loss with 4B.",
       "evidence": "Section 5.2 and Figure 5 show that increasing model size reduces the HR performance gap between natural and Unimax distributions.",
       "supported": "moderate"
     },
     {
       "claim": "The 2B model trained on natural data outperforms PaLM-Coder 8B on BC-HumanEval pass@100 for 12 of 14 languages, with average improvements of 48.17%.",
       "evidence": "Section 5.1 and Table 8 show pass@100 comparison. The authors attribute this partly to more training tokens and a filtered training dataset.",
       "supported": "moderate"
     },
     {
       "claim": "For translation tasks, uniformly balancing languages is the optimal strategy, as translation does not require natural language understanding from the training corpus.",
       "evidence": "Section 5.3 shows consistent improvements in test cases passed for both HR and LR languages on TP3 with balanced data. The paper argues this is because translation does not depend on diverse NL-code pairs.",
       "supported": "moderate"
     },
     {
       "claim": "For generation tasks, Unimax N=3 is optimal with respect to the tradeoff between LR gains and HR losses.",
       "evidence": "Section 5.2 reports Unimax 3 differences of 130.17%, 87.80%, and 36.00% (LR gain minus HR loss) for 1B, 2B, and 4B models on generation tasks.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "BabelCode enables execution-based evaluation of code models across 14 programming languages and 4 benchmarks. Training decoder-only models (1B-4B) on Unimax-balanced data distributions significantly improves low-resource language performance (average 66.48% better pass@k) at the cost of moderate high-resource degradation (12.94%), a tradeoff that improves with model scale. Translation tasks benefit more uniformly from balanced data than generation tasks, suggesting different dependencies on training corpus diversity.",
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are point estimates from single training runs with no error bars, confidence intervals, or variance across seeds. With 15 model configurations and hundreds of comparisons, the statistical reliability of observed differences is unknown."
     },
     {
       "flag": "No contamination analysis",
       "detail": "HumanEval, MBPP, and Transcoder are publicly available benchmarks whose solutions are widely shared on GitHub. The training corpus is curated from public GitHub code, but no decontamination or overlap analysis is performed."
     },
     {
       "flag": "Confounded baseline comparison",
       "detail": "The comparison between BC models and PaLM-Coder conflates multiple variables: different training data (filtered 14-language corpus vs. all GitHub), different training objectives (UL2 + causal LM vs. standard LM), different vocabulary (code-specific 64K vs. multilingual NL), and different training token counts. The 48.17% improvement claim does not isolate any single factor."
     },
     {
       "flag": "No limitations discussion",
       "detail": "The paper has no limitations or threats-to-validity section. Issues such as single-seed results, proprietary training data, limited model size range (1-4B), and potential benchmark contamination are not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and the HumanEval benchmark, a foundational code generation evaluation used as one of the four benchmarks in this paper."
     },
     {
       "title": "PaLM: Scaling language modeling with pathways",
       "authors": ["Aakanksha Chowdhery", "Sharan Narang", "Jacob Devlin"],
       "year": 2022,
       "arxiv_id": "2204.02311",
       "relevance": "PaLM-Coder serves as the primary external baseline for comparison; architecture decisions in this paper follow PaLM."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark used as one of the four evaluation datasets in this paper."
     },
     {
       "title": "A scalable and extensible approach to benchmarking nl2code for 18 programming languages",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "MultiPL-E is a concurrent multilingual code evaluation framework compared against BabelCode in Table 1."
     },
     {
       "title": "Multi-lingual evaluation of code generation models",
       "authors": ["Ben Athiwaratkun", "Sanjay Krishna Gouda", "Zijian Wang"],
       "year": 2023,
       "relevance": "MBXP is the most similar concurrent framework to BabelCode; detailed comparison in Sections 2.1-2.2."
     },
     {
       "title": "Competition-level code generation with alphacode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "Demonstrates decoder-only code models achieving competitive programming performance, relevant to the broader code LLM landscape."
     },
     {
       "title": "The Stack: 3 TB of permissively licensed source code",
       "authors": ["Denis Kocetkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2022,
       "arxiv_id": "2211.15533",
       "relevance": "Major open-source code dataset referenced for license filtering methodology and language distribution statistics."
     },
     {
       "title": "SantaCoder: don't reach for the stars!",
       "authors": ["Loubna Ben Allal", "Raymond Li", "Denis Kocetkov"],
       "year": 2023,
       "arxiv_id": "2301.03988",
       "relevance": "Contemporary open-source code LLM demonstrating decoder-only training on permissively licensed code."
     },
     {
       "title": "Unsupervised translation of programming languages",
       "authors": ["Baptiste Roziere", "Marie-Anne Lachaux", "Lowik Chanussot", "Guillaume Lample"],
       "year": 2020,
       "relevance": "Introduces the TransCoder benchmark and approach for code translation, used as one of the four evaluation datasets."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "First encoder-only model pre-trained on code, establishing the paradigm of pre-trained code models."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven C.H. Hoi"],
       "year": 2021,
       "relevance": "Encoder-decoder code model examining different pre-training strategies for code understanding and generation."
     },
     {
       "title": "Unified pre-training for program understanding and generation",
       "authors": ["Wasi Ahmad", "Saikat Chakraborty", "Baishakhi Ray", "Kai-Wei Chang"],
       "year": 2021,
       "relevance": "PLBART investigates data balancing strategies for code pre-training, the most directly related prior work on language distribution effects."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Execution-based code generation benchmark testing competitive programming capability of LLMs."
     }
   ]
 }