ai-research-survey

scan.json (30698B)

---

 {
   "paper": {
     "title": "Exploring Parameter-Efficient Fine-Tuning Techniques for Code Generation with Large Language Models",
     "authors": [
       "Martin Weyssow",
       "Xin Zhou",
       "Kisub Kim",
       "David Lo",
       "Houari Sahraoui"
     ],
     "year": 2024,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2308.10462",
     "doi": "10.1145/3714461"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PEFT techniques, especially LoRA, consistently outperform both ICL and RAG for Python code generation across multiple LLM families (CodeLlama, CodeGen2) and three benchmark datasets (Conala, CodeAlpacaPy, APPS). LLMs fine-tuned with LoRA using only ~1% of parameters significantly surpass fully fine-tuned SLMs by 39.8-72.3% in EM@k. QLoRA with 4-bit quantization achieves a 2x reduction in GPU memory while maintaining or improving effectiveness, enabling fine-tuning of models up to 34B parameters on a single 24GB GPU.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We make our code publicly available: https://github.com/martin-wey/peft-llm-code' in Section 4.6."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three datasets are public: Conala (curated version from Zhou et al. 2023), CodeAlpaca (Chaudhary 2023), and APPS (Hendrycks et al. 2021). The authors constructed CodeAlpacaPy by filtering Python samples from CodeAlpaca and release their code which includes this filtering."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using HuggingFace and PEFT libraries and an NVIDIA RTX A5000 24GB GPU, but does not provide library versions, a requirements.txt, Dockerfile, or detailed environment specification sufficient to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper provides detailed methodology in Section 4.6 and releases code on GitHub, the paper itself does not include step-by-step reproduction instructions or commands to replicate results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3, 4 and Figures 3-7 are reported as point estimates only. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses language like 'significantly outperform' and 'consistently outperform' throughout but performs no statistical significance tests. All comparative claims are based solely on numerical differences between point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently reports relative improvements with baseline context, e.g., 'surpasses the best small model by 39.8%, 41.7%, and 47.1%' (Section 5.2), '23.1% improvement in EM@10 on Conala (36.28 for LoRA vs. 29.47 for ICL)' (Section 5.3), and 'QLoRA-4bit results in a notable 52% increase' (Section 5.4)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for why the dataset sizes (Conala: 543 test, CodeAlpacaPy: 628 test, APPS: 750 test) are sufficient for the claims being made. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported for any experiment. Results appear to be from single runs with no indication of variability across seeds or runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive baselines are included: zero-shot, ICL with varying numbers of examples, RAG with GTE-small retrieval (Section 4.3), full fine-tuning for SLMs, and multiple PEFT techniques compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CodeLlama (2023), CodeGen2 (2023), and CodeT5+ (2023), which were recent models at the time of writing. RAG uses GTE-small, a competitive embedding model."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically compares LoRA, IA3, Prompt tuning, Prefix tuning, QLoRA-8bit, and QLoRA-4bit across the same models and datasets, effectively serving as an ablation across techniques. Effect of quantization precision (8-bit vs 4-bit) and number of ICL examples are also systematically varied."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Conala and CodeAlpacaPy use EM@1, EM@10, and CodeBLEU. APPS uses average test cases passed and Pass@k (k=1,2,5). Multiple metrics reported throughout."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is entirely automated (exact match, CodeBLEU, test case pass rates). No human evaluation of generated code quality is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All three datasets have separate train/validation/test splits. Section 4.6 states 'We selected the checkpoint with the lowest evaluation loss for inference,' confirming model selection on validation, reporting on test."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model, by PEFT technique, by dataset, and for APPS by difficulty level (introductory, interview, competition) in Table 4. Table 3 provides per-model per-technique breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where approaches fail: Prefix tuning 'fails to effectively adapt the larger models' (Section 5.2, some configurations show 0.0 EM@10 in Table 3), RAG underperforms ICL on CodeAlpacaPy (Section 5.3), and improvements are 'less substantial for interview and competition-level code generation' (Section 5.4)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Prefix tuning fails for larger models (0.0 EM scores in Table 3), increasing ICL examples beyond a threshold degrades performance (Section 5.1), RAG yields lower EM@10 than ICL on CodeAlpacaPy (Section 5.3), and QLoRA-8bit sometimes underperforms LoRA (Table 4)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims PEFT superiority over ICL and RAG, memory reduction with QLoRA, and broader applicability. All are supported by Tables 3-4 and Figures 1, 5-7. The abstract's claims about 'diverse set of LLMs' and 'three representative Python code generation datasets' accurately reflect the study scope."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims (e.g., 'LoRA significantly enhances performance', PEFT 'outperforms' alternatives) are based on controlled comparisons where only the tuning technique varies while model, data, and hardware are held constant. This controlled experimental design is adequate for the claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'Code Generation' broadly while testing only Python. The conclusion claims PEFT 'opens opportunities for broader applications of PEFT in software engineering scenarios' without evidence for non-Python languages or non-generation tasks. The threats-to-validity section acknowledges the monolingual limitation but then speculates that 'PEFT is also applicable to other programming languages.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section hypothesizes QLoRA-4bit's improvement 'stems from the regularization effect of reducing weight precision to 4 bits' (Section 6). The threats to validity section discusses hyperparameter sensitivity, model selection bias, and dataset representativeness as factors that could influence results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently frames its measurements (EM@k, CodeBLEU, Pass@k) as what they are — match-based and execution-based code generation metrics — without inflating them to broader claims about developer productivity or code quality. No proxy gap exists between measurements and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names and sizes are given: CodeLlama-7B, CodeLlama-7B-Instruct, CodeLlama-7B-Python, CodeLlama-13B-Python, CodeLlama-34B-Python, CodeGen2-1B/3.7B/7B, CodeGen-350M-mono, CodeT5+-220M/770M. For open-source models with fixed checkpoints, these names uniquely identify the model weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows the full prompt template with actual examples: '### Instruction: [intent] ### Response:'. The paper also describes the ICL prompt construction where examples are concatenated with the same template format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 provides detailed hyperparameters: learning rates (5e-5 for full FT, 3e-4 for LoRA/IA3/QLoRA, 3e-3/3e-2 for Prompt/Prefix tuning), r=16, alpha=32, 20 virtual tokens, Adafactor optimizer, 16-bit float, 5 epochs, batch size 8, beam search with beam size 10, max token lengths (64/128/1024)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is standard fine-tuning and inference with direct model prompting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes CodeAlpacaPy construction (filtering Python samples from CodeAlpaca, removing syntactically invalid code), Conala curation (ensured no function overlap between splits, no same-post overlap), and APPS split (4500/500/750 with 250 test samples per difficulty). Train/val/test sizes are stated for all datasets."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated, substantive discussion organized by external, internal, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: model selection limited to open-source (Section 7, external), monolingual Python-only evaluation (external), hyperparameter values taken from prior work without exhaustive tuning (internal), CodeBLEU's reliance on dataflow graphs not always available for small examples (construct), lack of unit tests in Conala/CodeAlpacaPy (construct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: closed-source models excluded due to parameter inaccessibility, full fine-tuning infeasible for LLMs within 24GB GPU constraint, HumanEval/MBPP excluded for lacking training data, Python only (acknowledging monolingual limitation), and no larger models beyond QLoRA experiments."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the input datasets are public and code is released, raw experimental outputs (model predictions, per-example scores) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes how each dataset was collected: Conala crawled from StackOverflow with manual annotations, CodeAlpacaPy filtered from CodeAlpaca for Python with syntactic validation, APPS from coding competitions with difficulty categorization."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (Conala from StackOverflow, CodeAlpaca from LLM-generated data, APPS from coding competitions)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: CodeAlpacaPy was constructed by filtering Python samples from CodeAlpaca and removing syntactically invalid code. Dataset splits are specified with exact counts (e.g., Conala: 2,135/201/543). The Conala curation process ensuring no overlap between splits is described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers appear in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: DIRO, University of Montreal (Weyssow, Sahraoui) and Singapore Management University (Zhou, Kim, Lo). These are academic institutions with no apparent conflict with the evaluated open-source models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Without funding disclosure, independence of the funder cannot be assessed. The authors are at academic institutions and evaluate open-source models they did not develop, suggesting likely independence, but this is not explicitly stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (CodeLlama, CodeGen2, CodeGen, CodeT5+). The paper mentions CodeLlama is based on Llama 2 and CodeGen2 was pre-trained on TheStack, but does not state when training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses train/test split integrity for fine-tuning (Conala curated to avoid function overlap), but does not discuss whether pretrained models may have seen test examples during pretraining. Conala (StackOverflow) and APPS (competition problems) are publicly available and could appear in pretraining corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All three benchmarks (Conala from 2018, CodeAlpaca from 2023, APPS from 2021) were available online before the models' training periods. The paper does not discuss whether models may have seen benchmark solutions during pretraining."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments are automated benchmark evaluations of code generation models."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on existing public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper focuses extensively on training memory consumption (Figures 1 and 5) but does not report inference cost, latency, or time per example for any configuration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states hardware used (single NVIDIA RTX A5000 24GB GPU) and peak memory consumption, but does not report total GPU hours, wall-clock training time, or total computational budget across all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. For ICL, the paper mentions 'selecting random few-shot examples using different seeds' (Section 8) but does not report results across seeds. Fine-tuning results appear to be single-seed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs with no indication of how many trials produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Section 7 (internal validity) states 'we used hyperparameters values which have been used in previous work' but does not report whether any search was performed or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 clearly states: 'We selected the checkpoint with the lowest evaluation loss for inference,' which is a principled selection on the validation set, not the test set."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all techniques (LoRA, IA3, Prompt tuning, Prefix tuning, ICL, RAG) themselves using the PEFT library. They do not acknowledge potential bias in their own implementations of these methods or discuss whether their ICL/RAG implementations are competitive."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 explicitly categorizes techniques by computation cost vs effectiveness. Figure 5 shows EM@10 alongside GPU peak memory for different quantization levels. Figure 1 compares memory consumption across fine-tuning approaches. The computation-effectiveness tradeoff is a central theme."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Conala, CodeAlpacaPy, or APPS actually measure meaningful code generation capability. EM@k requires exact string match to reference solutions, a very narrow proxy for code quality, but this limitation is not discussed. The construct validity threats in Section 7 address only metric choice, not benchmark validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models are evaluated with the same direct inference setup (prompt in, code out), so no scaffold confound exists."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Conala (2018) and APPS (2021) predate the models used (CodeLlama and CodeGen2, both 2023). Models could have seen benchmark solutions during pretraining, but temporal leakage is not mentioned."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The Conala dataset includes variable hints in natural language intents (e.g., 'map two lists keys and values into a dictionary'), which provide ground-truth variable names, but this is not analyzed as potential feature leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes Conala's curated version ensures no function overlap between fine-tuning splits and no same-post overlap, but does not address whether pretrained models' training corpora overlap with benchmark data. StackOverflow data (Conala) is commonly included in code pretraining corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference tests, n-gram overlap analysis, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "ICL drastically improves effectiveness of all models compared to zero-shot, with CodeLlama-7B achieving EM@10 of 29.83 (vs 7.73 zero-shot) on Conala.",
       "evidence": "Figure 3 shows EM@10 scores across models with 0-16 ICL examples on Conala and CodeAlpacaPy. All models show large gains from zero-shot to ICL. Section 5.1 reports specific numbers.",
       "supported": "strong"
     },
     {
       "claim": "LLMs fine-tuned with PEFT consistently and significantly outperform fully fine-tuned SLMs by 39.8-72.3% in EM@k under the same 24GB GPU constraint.",
       "evidence": "Table 3 shows CodeLlama-7B-Python with LoRA (EM@10: 36.28 on Conala, 15.92 on CodeAlpacaPy) vs CodeGen-350M-mono full FT (EM@10: 18.42 on Conala, 5.73 on CodeAlpacaPy). Section 5.2 reports the percentage improvements.",
       "supported": "strong"
     },
     {
       "claim": "LoRA is the most effective PEFT technique among LoRA, IA3, Prompt tuning, and Prefix tuning across the studied models.",
       "evidence": "Table 3 shows LoRA achieving the highest or near-highest scores for most model-dataset combinations. Section 5.2 states 'LoRA emerges as the most effective PEFT technique among the studied ones.'",
       "supported": "strong"
     },
     {
       "claim": "QLoRA-4bit achieves a 2x reduction in peak memory usage compared to LoRA while improving or preserving effectiveness.",
       "evidence": "Figure 5 shows CodeLlama-7B-Python LoRA at ~19GB vs QLoRA-4bit at ~9GB, with QLoRA-4bit achieving comparable or better EM@10 scores. CodeLlama-34B-Python fine-tuned with QLoRA-4bit fits within 24GB.",
       "supported": "moderate"
     },
     {
       "claim": "LoRA significantly outperforms both ICL and RAG across CodeLlama-7B variants on Conala and CodeAlpacaPy.",
       "evidence": "Figure 6 shows all models achieving higher EM@10 with LoRA vs ICL. Figure 7 shows CodeLlama-7B LoRA (39.31 EM@10) vs RAG peak (35.17) vs ICL (29.83) on Conala. Section 5.3 provides detailed comparisons.",
       "supported": "moderate"
     },
     {
       "claim": "QLoRA-4bit boosts CodeLlama-7B-Instruct's average passed test cases on APPS introductory problems by 52% over the base model.",
       "evidence": "Table 4 shows average passed tests increasing from 13.66 (base) to 20.84 (QLoRA-4bit) for introductory level, and from 13.44 to 20.34 for interview level. Pass@5 increases from 8.80 to 12.40 at introductory level.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or uncertainty quantification",
       "detail": "All results across all experiments appear to be from single runs. No standard deviations, confidence intervals, or repeated-run statistics are reported despite the paper making strong comparative claims. Results could shift substantially with different random seeds."
     },
     {
       "flag": "Statistical significance claims without tests",
       "detail": "The paper repeatedly uses language like 'significantly outperform' and 'LoRA significantly enhances' without performing any statistical significance tests. All comparative conclusions rest on point-estimate comparisons."
     },
     {
       "flag": "No contamination analysis for pretrained models",
       "detail": "All benchmark datasets (Conala from StackOverflow 2018, APPS from competitions 2021) were publicly available before the models' training data collection. CodeLlama and CodeGen2 likely encountered StackOverflow content during pretraining, potentially inflating zero-shot and ICL baselines."
     },
     {
       "flag": "No total compute budget reported",
       "detail": "Despite running experiments across 11 models, 6 PEFT techniques, and 3 datasets, the paper does not report total GPU hours, training time, or wall-clock time for any experiment. Peak memory is reported but total computational cost is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced Codex and HumanEval benchmark, foundational work on LLM code generation evaluation."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2023,
       "arxiv_id": "2203.13474",
       "relevance": "Open-source code LLM family used as baseline; demonstrates scaling effects for code generation."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Primary LLM family evaluated; open foundation model for code demonstrating instruction tuning and specialization."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "Introduced LoRA, the best-performing PEFT technique in this study; foundational for efficient fine-tuning of LLMs."
     },
     {
       "title": "QLoRA: Efficient finetuning of quantized LLMs",
       "authors": ["Tim Dettmers", "Artidoro Pagnoni", "Ari Holtzman"],
       "year": 2023,
       "arxiv_id": "2305.14314",
       "relevance": "Introduced QLoRA combining LoRA with quantization; enables fine-tuning larger models on consumer GPUs."
     },
     {
       "title": "CodeT5+: Open code large language models for code understanding and generation",
       "authors": ["Yue Wang", "Hung Le", "Akhilesh Deepak Gotmare"],
       "year": 2023,
       "arxiv_id": "2305.07922",
       "relevance": "Encoder-decoder code LLM used as SLM baseline; demonstrates pre-training objectives for code intelligence."
     },
     {
       "title": "DocPrompting: Generating code by retrieving the docs",
       "authors": ["Shuyan Zhou", "Uri Alon", "Frank F Xu"],
       "year": 2023,
       "relevance": "RAG approach for code generation; provides curated Conala dataset version used in this study."
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Introduced APPS benchmark with execution-based evaluation of code generation at varying difficulty levels."
     },
     {
       "title": "A Systematic Evaluation of Large Language Models of Code",
       "authors": ["Frank F. Xu", "Uri Alon", "Graham Neubig"],
       "year": 2022,
       "doi": "10.1145/3520312.3534862",
       "relevance": "Systematic evaluation of code LLMs providing baseline comparisons and evaluation methodology."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Early work on LLM program synthesis including MBPP benchmark; demonstrates zero-shot code generation capability."
     },
     {
       "title": "Few-shot parameter-efficient fine-tuning is better and cheaper than in-context learning",
       "authors": ["Haokun Liu", "Derek Tam", "Mohammed Muqeeth"],
       "year": 2022,
       "relevance": "Introduced IA3 PEFT technique; directly relevant comparison showing PEFT advantages over ICL in NLP."
     },
     {
       "title": "Execution-Based Evaluation for Open-Domain Code Generation",
       "authors": ["Zhiruo Wang", "Shuyan Zhou", "Daniel Fried"],
       "year": 2022,
       "arxiv_id": "2212.10481",
       "relevance": "Execution-based evaluation methodology for code generation; provides context on evaluation challenges."
     }
   ]
 }