ai-research-survey

scan.json (29192B)

---

 {
   "paper": {
     "title": "PanGu-Coder2: Boosting Large Language Models for Code with Ranking Feedback",
     "authors": [
       "Bo Shen",
       "Jiaxin Zhang",
       "Taihong Chen",
       "Daoguang Zan",
       "Bing Geng",
       "An Fu",
       "Muhan Zeng",
       "Ailun Yu",
       "Jichuan Ji",
       "Jingyang Zhao",
       "Yuenan Guo",
       "Qianxiang Wang"
     ],
     "year": 2023,
     "venue": "arXiv preprint",
     "arxiv_id": "2307.14936",
     "doi": "10.48550/arXiv.2307.14936"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PanGu-Coder2, a 15B parameter Code LLM fine-tuned with the RRTF framework on StarCoder, achieves 62.20% pass@1 on HumanEval, outperforming all prior open-source Code LLMs including WizardCoder (59.80%). The model shows consistent improvements across HumanEval, CoderEval, and LeetCode, with a 28% absolute pass@1 improvement over the StarCoder base model. CTranslate2 int8 quantization halves memory usage and doubles inference speed with no performance loss.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper references StarCoder and RRHF code repositories but does not release the PanGu-Coder2 code, training scripts, or RRTF implementation."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The curated 68K training dataset is not released. Section 3.3 describes starting from the public CodeAlpaca-20k dataset and evolving it, but the resulting corpus is not made available. The evaluation benchmarks (HumanEval, CoderEval, LeetCode) are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 lists model architecture hyperparameters (hidden size, attention heads, layers) but no environment specifications such as library versions, CUDA version, requirements.txt, or Dockerfile are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper describes the RRTF framework at a conceptual level but does not provide runnable scripts or detailed reproduction guides."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2, 3, and 4 are reported as point estimates without confidence intervals or error bars. No uncertainty quantification is provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims PanGu-Coder2 'outperforms' multiple baselines across all benchmarks but provides no statistical significance tests. All comparisons are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context: '28% absolute improvement in terms of pass@1 score (from 33.6% to 61.6%)' over StarCoder (Section 4.2.1), and '4.34%' improvement over WizardCoder. Both baseline and new values are given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why HumanEval (164 problems), CoderEval (230 functions), or LeetCode (300 problems) are adequate sample sizes for the claims being made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported across training runs. Results appear to come from a single training run. The n=200 samples for pass@k estimation provide sampling diversity but no across-run variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2 and 3 compare against multiple baselines: CodeGen-mono 16B, CodeGeeX 13B, StarCoder 15B, CodeT5+ 16B, WizardCoder 15B, and closed-source models including GPT-3.5 and GPT-4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "WizardCoder (2023), StarCoder (2023), CodeT5+ (2023) are contemporary to the paper. The paper states WizardCoder was 'the state-of-the-art Code LLM prior to PanGu-Coder2.'"
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows performance vs. dataset size (18k, 38k, 68k) and training steps, but there is no ablation of the RRTF framework components — e.g., removing ranking loss, removing teacher feedback, comparing against plain SFT with the same data. The contribution of RRTF vs. data quality is not isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported for pass@1, pass@10, and pass@100 (Table 2), as well as greedy decoding pass@1 on three different benchmarks (Table 3). Table 4 reports GPU memory, inference speed, and HumanEval performance for quantization."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated code is conducted. Figures 6-8 show case studies with manual inspection but these are qualitative examples, not systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval, CoderEval, and LeetCode are separate test benchmarks not used during training. Section 3.3 explicitly states they checked for overlap between the 68K training data and HumanEval. LeetCode uses problems after July 2022, post-dating StarCoder's training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 breaks down LeetCode performance by difficulty (easy/medium/hard: 32/30/10). Figure 5 shows Venn diagrams of solved problems by model. Results across three distinct benchmarks provide task-level diversity."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 shows a case where all three models (PanGu-Coder2, WizardCoder, StarCoder) generate incorrect code. Figure 7 shows a case where WizardCoder outperforms PanGu-Coder2. Section 4.2.3 discusses strengths and weaknesses."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows WizardCoder outperforming PanGu-Coder2 on certain problems. Table 4 shows GPTQ quantization significantly degrades HumanEval performance (62.20% → 51.22%). Figure 5 shows 18 problems none of the models can solve even with 200 samples."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 62.20% pass@1 on HumanEval, confirmed in Table 3. The claim of outperforming 'all previous Code LLMs' is supported by Tables 2 and 3 across three benchmarks, with the caveat that GPT-4 (a general LLM, not a 'Code LLM') scores higher."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RRTF 'effectively and efficiently boost[s]' code generation. However, there is no ablation separating the effects of RRTF from the curated 68K dataset, the Evol-Instruct data augmentation, or teacher model knowledge distillation. The causal role of the ranking feedback mechanism is not isolated."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper's title claims to 'Boost Large Language Models for Code' but evaluation is exclusively on Python. The abstract claims RRTF 'can effectively and efficiently boost pre-trained large language models for code generation' without bounding this to Python or the specific model tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The improvement could stem from the curated training data quality, teacher model knowledge transfer, the Evol-Instruct augmentation, or RRTF specifically — but these are never disentangled or discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on specific benchmarks and frames results in those terms. It does not overclaim that pass@k on HumanEval represents general 'programming ability' or 'code quality.' Claims are tied to the specific metrics measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The base model is identified as 'StarCoder 15B' without a specific checkpoint or version. The teacher models used in RRTF training are never identified by name or version — Section 3.4 refers only to generic 'teacher models.' GPT-3.5/GPT-4 comparison scores are cited from other papers without version specification."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.5 provides the exact evaluation prompts used for PanGu-Coder2, StarCoder, and WizardCoder. Figure 2 provides the Evol-Instruct prompt template. Figure 3 shows the training data format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists model architecture hyperparameters. Section 3.5 states 'global batch size of 512 for 6 epochs.' Section 4.1.4 specifies temperature=0.2 for pass@1, temperature=1.2 for pass@10/100, top_p=0.95, max_new_tokens=512."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models directly generate code from prompts without multi-step reasoning, tool use, or agent frameworks."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 mentions 'we conducted data preprocessing on our initial corpus using several manually-defined rules and reduced the size of the corpus to 68K' but never specifies what those rules were or how many examples were removed at each step."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section or threats-to-validity discussion in the paper. The conclusion mentions future work but does not discuss limitations of the current study."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not acknowledge that results are limited to Python, to the specific benchmarks tested, or to the 15B model scale."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the 68K training dataset nor per-problem evaluation results are made available. Only aggregate metrics are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.3 describes the high-level pipeline (Alpaca 20K → Evol-Instruct → 100K → filtering → 68K) but omits critical details: what teacher models were sampled, what the filtering rules were, and what constituted the removed 32K samples."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are the CodeAlpaca-20k public dataset and standard code generation benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from 100K to 68K involves unspecified 'manually-defined rules.' The ranking and scoring procedure (Section 3.4) is described at a high level but lacks specifics on how many samples fell into each category (compiled error, runtime error, partial pass, all pass)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Huawei Cloud Co., Ltd., Chinese Academy of Science, and Peking University. The affiliation header identifies which authors belong to each institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The majority of authors are from Huawei Cloud, which has a commercial interest in demonstrating strong code LLM performance. Huawei is not independent of the outcome — a successful PanGu-Coder2 promotes their commercial interests."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 states StarCoder 'only consists of code before June 2022.' This establishes the base model's training data cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3: 'we devoted considerable efforts to surveying the potential overlap between the collected 68K dataset and the HumanEval benchmark. After conducting a meticulous survey, we confirm that there is no data leakage in our experiments.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval was published in July 2021, and StarCoder's training data includes code before June 2022 — solutions could have been posted to GitHub in that window. The paper only checks overlap with the 68K fine-tuning data, not with StarCoder's pre-training corpus. For LeetCode they create a temporal split, but HumanEval contamination through the base model is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports GPU memory consumption (32.36 GB for float16) and inference speed (75 ms/token) for PanGu-Coder2 and its quantized variants."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total training compute budget is stated. The paper mentions batch size 512 for 6 epochs on 68K samples, but does not report GPU hours, hardware used for training, or total training time."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. Results appear to come from a single training run with single-seed evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of training runs is not stated. For inference, n=200 samples are generated per problem for pass@k, but it is unclear if this represents a single training checkpoint or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper does not describe how training hyperparameters (batch size 512, 6 epochs) were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Table 2 states results are the 'optimal pass@1/10/100' across temperature settings, following Codex methodology. However, the training configuration selection (why 68K, why 6 epochs) is not justified through validation-set selection — Figure 4 shows training curves but doesn't indicate which checkpoint was selected or how."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Huawei Cloud authors evaluate their own PanGu-Coder2 model without acknowledging self-evaluation bias. Baseline scores for some models are taken from other papers, while PanGu-Coder2 is evaluated by the authors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "PanGu-Coder2 receives additional RRTF training on 68K examples beyond StarCoder's pre-training, but no compute-matched comparison is provided. The additional training cost vs. baseline models like WizardCoder is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, CoderEval, or LeetCode adequately measure real-world code generation ability. The benchmarks are used without questioning their construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1.5 acknowledges 'performance of a Code LLM could be largely affected by the prompt used' and uses different prompts per model (each model's recommended prompt). However, this prompt variation is a confound that is not controlled for — performance differences could partially reflect prompt format advantages rather than model quality."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For LeetCode, the paper explicitly creates a temporal split: problems created after July 1, 2022, post-dating StarCoder's training data cutoff (before June 2022). Section 4.1.2 states this 'ensures that any data in this benchmark does not overlap with the training data of StarCoder.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether evaluation prompts or test harness details could leak answer information. The different prompt formats per model (Section 4.1.5) could provide varying amounts of contextual information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training examples and benchmark problems share structural similarities (e.g., common coding patterns, similar problem types from the same sources)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states: 'we devoted considerable efforts to surveying the potential overlap between the collected 68K dataset and the HumanEval benchmark.' This is a concrete overlap detection step, though limited to the fine-tuning data and one benchmark."
       }
     }
   },
   "claims": [
     {
       "claim": "PanGu-Coder2 achieves 62.20% pass@1 on HumanEval with greedy decoding, and 61.64% with temperature sampling (n=200)",
       "evidence": "Tables 2 and 3 report these results. Section 4.1.4 describes the evaluation methodology following Codex/AlphaCode conventions.",
       "supported": "moderate"
     },
     {
       "claim": "PanGu-Coder2 outperforms all previously published Code LLMs on HumanEval, CoderEval, and LeetCode",
       "evidence": "Tables 2 and 3 show PanGu-Coder2 achieving the best open-source results across all three benchmarks. It also outperforms GPT-3.5 on HumanEval but not GPT-4.",
       "supported": "moderate"
     },
     {
       "claim": "RRTF provides a 28% absolute improvement over StarCoder base model on HumanEval pass@1",
       "evidence": "Table 2: StarCoder 33.60% → PanGu-Coder2 61.64% (sampling). Table 3: StarCoder 32.93% → PanGu-Coder2 62.20% (greedy). The improvement is real but conflated with data quality effects.",
       "supported": "moderate"
     },
     {
       "claim": "RRTF is simpler and more efficient than PPO-based RL approaches for code generation",
       "evidence": "Section 1 argues this conceptually, citing PPO complexity. No direct empirical comparison of training cost, convergence speed, or implementation complexity against PPO-based methods.",
       "supported": "weak"
     },
     {
       "claim": "Larger dataset size leads to better and more stable training",
       "evidence": "Figure 4 shows training curves for 18k, 38k, and 68k datasets. Performance improves and stabilizes with more data. However, only three data points with no controlled comparison.",
       "supported": "moderate"
     },
     {
       "claim": "CTranslate2 int8 quantization slightly improves HumanEval performance (62.20% → 64.63%)",
       "evidence": "Table 4 reports this result with a single run. No error bars or multiple runs to confirm the improvement is not noise. The authors themselves call it 'incredible' and plan further study.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "Huawei Cloud authors evaluate PanGu-Coder2 without independent validation or acknowledgment of self-evaluation bias. All evaluation was conducted by the team that built the model."
     },
     {
       "flag": "No statistical tests on any comparison",
       "detail": "All claims of 'outperforms' are based on comparing point estimates from what appears to be a single training run with no error bars. The 4.34% improvement over WizardCoder could easily be within noise."
     },
     {
       "flag": "Confounded attribution",
       "detail": "The improvement is attributed to RRTF but could stem from the curated 68K dataset, teacher model knowledge, Evol-Instruct augmentation, or their combination. No ablation isolates the RRTF contribution."
     },
     {
       "flag": "Unidentified teacher models",
       "detail": "The teacher models used in RRTF training are never named. If teacher models include GPT-4 or similar powerful models, much of the improvement could be knowledge distillation rather than the RRTF framework."
     },
     {
       "flag": "Different prompts across models",
       "detail": "Section 4.1.5 shows each model uses a different prompt format. PanGu-Coder2 uses docstring-first format while WizardCoder uses instruction format. Performance differences could partly reflect prompt format advantages."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations, threats to validity, or scope boundaries despite making broad claims about Code LLM boosting."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and HumanEval benchmark, the primary evaluation benchmark used in this paper and widely in code LLM research."
     },
     {
       "title": "StarCoder: may the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Base model for PanGu-Coder2; 15B open-source code LLM that is the starting point for RRTF fine-tuning."
     },
     {
       "title": "WizardCoder: Empowering code large language models with evol-instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Prior state-of-the-art Code LLM on HumanEval; key baseline and source of Evol-Instruct technique used in PanGu-Coder2."
     },
     {
       "title": "CodeRL: Mastering code generation through pretrained models and deep reinforcement learning",
       "authors": ["Hung Le", "Yue Wang", "Akhilesh Deepak Gotmare"],
       "year": 2022,
       "arxiv_id": "2207.01780",
       "relevance": "Pioneered RL-based code generation with actor-critic framework and unit test signals; key prior work on RL for code LLMs."
     },
     {
       "title": "RLTF: Reinforcement learning from unit test feedback",
       "authors": ["Jiate Liu", "Yiqin Zhu", "Kaiwen Xiao"],
       "year": 2023,
       "relevance": "Online RL framework with multi-granularity unit test feedback for code generation; direct competitor to RRTF approach."
     },
     {
       "title": "Execution-based code generation using deep reinforcement learning",
       "authors": ["Parshin Shojaee", "Aneesh Jain", "Sindhu Tipirneni"],
       "year": 2023,
       "arxiv_id": "2301.13816",
       "relevance": "PPOCoder: PPO-based RL for code generation that showed limited improvements on MBPP benchmark."
     },
     {
       "title": "RRHF: Rank responses to align language models with human feedback without tears",
       "authors": ["Zheng Yuan", "Hongyi Yuan", "Chuanqi Tan"],
       "year": 2023,
       "arxiv_id": "2304.05302",
       "relevance": "Core inspiration for RRTF framework; proposed ranking-based alignment as simpler alternative to RLHF."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF foundational work that inspired the ranking-based feedback approach adapted in RRTF."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "doi": "10.48550/arXiv.2303.08774",
       "relevance": "Key commercial comparison point; GPT-4 outperforms PanGu-Coder2 on HumanEval."
     },
     {
       "title": "Textbooks are all you need",
       "authors": ["Suriya Gunasekar", "Yi Zhang", "Jyoti Aneja"],
       "year": 2023,
       "arxiv_id": "2306.11644",
       "relevance": "phi-1: data-efficient code LLM achieving strong results with only 1.3B parameters through data quality."
     },
     {
       "title": "CodeT5+: Open code large language models for code understanding and generation",
       "authors": ["Yue Wang", "Hung Le", "Akhilesh Deepak Gotmare"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Encoder-decoder Code LLM baseline with modular architecture for diverse code tasks."
     },
     {
       "title": "CoderEval: A benchmark of pragmatic code generation with generative pre-trained models",
       "authors": ["Hao Yu", "Bo Shen", "Dezhi Ran"],
       "year": 2023,
       "arxiv_id": "2302.00288",
       "relevance": "Context-aware code generation benchmark used to evaluate PanGu-Coder2 beyond HumanEval's simple function generation."
     }
   ]
 }