ai-research-survey

scan-v5.json (27550B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Exploring Parameter-Efficient Fine-Tuning Techniques for Code Generation with Large Language Models",
     "authors": [
       "M. Weyssow",
       "Xin Zhou",
       "Kisub Kim",
       "David Lo",
       "H. Sahraoui"
     ],
     "year": 2023,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2308.10462",
     "doi": "10.1145/3714461"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims PEFT superiority over ICL/RAG and QLoRA memory reduction are directly supported by Tables 3-4 and Figures 5-7 showing EM@k and GPU memory results across all model families.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'LoRA improves effectiveness' are supported by controlled comparisons holding models constant and varying technique across identical datasets and splits; the design is adequate for comparative causal claims.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds claims to Python code generation, single-GPU resource constraint, and the specific model families tested; Threats to Validity (Section 7) explicitly flags the monolingual limitation and restricted model selection.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The main finding that PEFT beats ICL/RAG is not accompanied by discussion of alternative explanations (e.g., whether optimized ICL example selection would close the gap); only the QLoRA-4bit improvement mentions a hypothesis (regularization effect).",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses EM@k, CodeBLEU, and Pass@k as proxies for code generation quality and explicitly notes in Section 5.3 the distinction between EM (requiring exact match) and CodeBLEU (rewarding near-correct solutions), clarifying what each metric captures.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' contains dedicated subsections for external, internal, and construct validity with multiple specific threats discussed.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: Python-only datasets limiting multilingual generalizability, hyperparameter choices based on prior work without sensitivity analysis, and EM@k not capturing execution correctness for Conala/CodeAlpacaPy.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it excludes closed-source models, excludes full fine-tuning for LLMs due to resource constraints, and notes that combining ICL/RAG with fine-tuned LLMs was not explored.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or disclosure appears anywhere in the provided paper text.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors' institutional affiliations (University of Montreal, Singapore Management University) are disclosed on the title page with email addresses.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "Funding is not disclosed, making independence assessment impossible.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "PEFT, ICL, RAG, LLM (≥1B parameters), and SLM (<1B parameters) are all explicitly defined in Sections 1-2 with precise parameter-count boundaries and technical descriptions.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 explicitly lists three contributions: comprehensive empirical study of 6 PEFT techniques for LLMs in code generation, comparison against ICL/RAG, and demonstration of practicality under limited resources.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly distinguishes this work from prior PEFT studies by noting they focused on SLMs (<0.25B parameters) and explicitly claims this is 'among the first comprehensive exploration of PEFT techniques for LLMs in software engineering.'",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "Code is publicly available at https://github.com/martin-wey/peft-llm-code, mentioned in Section 4.6.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Conala, APPS, and CodeAlpaca are all publicly available datasets; CodeAlpacaPy is a filtered subset of CodeAlpaca described in sufficient detail to reproduce.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only the GPU model (NVIDIA RTX A5000 24GB) and library names (HuggingFace, PEFT) are mentioned; no requirements.txt, Dockerfile, or versioned dependency list is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions appear in the paper; the hyperparameters are listed but no runnable workflow or README-equivalent is described.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 3-4 and Figures 3-7 are reported as single point estimates with no confidence intervals or error bars.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to any comparative claims despite the paper making numerous 'X outperforms Y' conclusions across all four RQs.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Percentage improvements are reported with baselines (e.g., 'best LLM surpasses best small model by 39.8–72.3% in EM@k', 'QLoRA-4bit boosting average passed tests by 52%') providing effect size context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "Dataset sizes are described but no power analysis or justification for why 543/628/750 test examples are sufficient to detect the observed effect sizes is provided.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All reported EM@k and CodeBLEU scores are single values with no standard deviation, variance, or multi-run averaging reported.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Zero-shot, ICL (random), RAG, and full fine-tuning for SLMs are all used as baselines against PEFT techniques.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "CodeLlama (2023), CodeGen2 (2023), and CodeT5+ (2023) are all recent model families; RAG uses GTE-small described as outperforming OpenAI embeddings.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The systematic comparison across LoRA, IA3, Prompt tuning, Prefix tuning, QLoRA-8bit, and QLoRA-4bit effectively ablates the contribution of each PEFT design choice.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "EM@1, EM@10, CodeBLEU are used for Conala/CodeAlpacaPy; average test cases passed and Pass@k (k=1,2,5) are used for APPS.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Automated code generation benchmarks with ground truth make human evaluation not clearly required for the claims made; the paper focuses on match-based and execution-based correctness.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "All three datasets have explicit train/validation/test splits; Conala 2135/201/543, CodeAlpacaPy 2192/314/628, APPS 4500/500/750.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 breaks APPS results into introductory, interview, and competition difficulty levels; model family breakdowns across SLMs and LLMs are provided throughout.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper notes that improvements are 'less substantial for interview and competition-level tasks' and that Prefix tuning 'fails to effectively adapt larger models,' but no specific failure case examples are shown.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Negative results are clearly reported: Prefix tuning fails for larger LLMs, RAG underperforms ICL on complex CodeAlpacaPy, and PEFT gains are minimal for competition-level APPS problems.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model variants are named: CodeGen-350M-mono, CodeT5+-220M/770M, CodeGen2-1B/3.7B/7B, CodeLlama-7B/7B-Instruct/7B-Python/13B-Python/34B-Python with exact parameter counts.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 shows the actual prompt template with '### Instruction:' and '### Response:' delimiters plus three concrete examples from each dataset.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.6 reports learning rates (5e-5 for full FT, 3e-4 for LoRA/IA3/QLoRA, 3e-3 for Prompt tuning, 3e-2 for Prefix tuning), LoRA rank r=16, alpha=32, 20 virtual tokens, batch size 8, 5 epochs, beam size 10.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "This is a fine-tuning/inference study with no agentic scaffolding; the question is not applicable.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "CodeAlpacaPy construction is described (filtering for Python, static parsing for syntactic validity); Conala curation is described (ensuring StackOverflow post separation across splits, function uniqueness).",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "All three datasets (Conala, APPS, CodeAlpaca) are publicly available; the filtered CodeAlpacaPy subset is derivable from the public CodeAlpaca dataset using the described procedure.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 describes each dataset's origin: Conala crawled from StackOverflow with manual annotation, APPS from competitive programming, CodeAlpacaPy filtered from CodeAlpaca for syntactically valid Python.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; all data from standard benchmarks and code repositories.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from dataset selection through train/val/test splitting, preprocessing, fine-tuning, and evaluation is described in Sections 4.2-4.6.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff dates are stated for CodeLlama, CodeGen2, or CodeT5+ despite these models having known pre-training corpora that may overlap with benchmark datasets.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Intra-dataset train/test overlap is addressed for Conala, but whether model pre-training data (TheStack, code data) contains the APPS, Conala, or CodeAlpaca test examples is not discussed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Conala (2018), APPS (2021), and CodeAlpaca (2023) were available before CodeLlama and CodeGen2 training cutoffs; this potential contamination is not addressed.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Peak GPU memory consumption during inference and fine-tuning is reported in Figure 1 for all model configurations, which is the primary resource constraint discussed.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "The entire study is explicitly conducted under a single NVIDIA RTX A5000 24GB GPU constraint, stated as the computational budget in Section 4.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "PEFT techniques (LoRA, IA3) consistently outperform ICL for LLMs on code generation",
       "evidence": "Figure 6 shows all models fine-tuned with LoRA achieve significantly higher EM@10 than their ICL counterparts on both Conala and CodeAlpacaPy; CodeLlama-7B-Python LoRA achieves 36.28 vs 29.47 ICL EM@10 on Conala (23.1% improvement)",
       "supported": "strong"
     },
     {
       "claim": "LLMs fine-tuned with PEFT outperform SLMs fully fine-tuned by 39.8–72.3% in EM@k",
       "evidence": "Table 3 shows best LLM (CodeLlama-7B-Python with LoRA) vs best SLM (CodeGen-350M-mono with LoRA): 39.8–72.3% improvement in EM@k on Conala and CodeAlpacaPy under same 24GB GPU constraint",
       "supported": "strong"
     },
     {
       "claim": "QLoRA-4bit reduces peak GPU memory up to 2x versus LoRA while maintaining effectiveness",
       "evidence": "Figure 1 shows CodeLlama-7B-Python: LoRA uses 19.06GB, QLoRA-4bit uses 9.16GB (2x reduction); Figure 5 shows QLoRA-4bit achieves 40.70 EM@10 vs LoRA's 36.28 on Conala for CodeLlama-34B",
       "supported": "strong"
     },
     {
       "claim": "LoRA outperforms RAG for code generation on both datasets across all CodeLlama variants",
       "evidence": "Figure 7 shows CodeLlama-7B achieves 39.31 EM@10 with LoRA vs 35.17 with RAG (best) vs 29.83 with ICL on Conala; similar pattern holds for CodeAlpacaPy",
       "supported": "strong"
     },
     {
       "claim": "PEFT outperforms full fine-tuning for SLMs, contrasting with NLP findings",
       "evidence": "Table 3 shows CodeGen-350M-mono LoRA achieves 25.60 EM@10 on Conala vs 18.42 for full fine-tuning; similar patterns for CodeT5+ variants. Authors note this contrasts with Ding et al.'s NLP finding that full fine-tuning is superior",
       "supported": "strong"
     },
     {
       "claim": "Prefix tuning fails to effectively adapt larger LLMs to code generation datasets",
       "evidence": "Table 3 shows Prefix tuning yields 0.0 EM@1 and 0.16–0.32 EM@10 on CodeAlpacaPy for CodeGen2-7B, CodeLlama variants, and all models ≥3.7B, while LoRA achieves 7–8% EM@1 on the same models",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "PEFT techniques, particularly LoRA, consistently outperform both ICL and RAG for Python code generation across 11 LLMs and SLMs tested under a single 24GB GPU constraint. LLMs fine-tuned with PEFT surpass fully fine-tuned SLMs by 39–72% in EM@k, and PEFT also beats full fine-tuning for SLMs (contrasting with NLP literature). QLoRA-4bit enables fine-tuning of 34B parameter models within a 24GB GPU while achieving comparable or superior performance to LoRA, and Prefix tuning consistently fails for models above 3.7B parameters. Benchmark contamination from model pre-training data is unaddressed, and no statistical significance tests are applied to any comparative claims.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('LoRA significantly enhances', 'consistently outperforms') are made without any statistical tests; single-run point estimates are reported throughout Tables 3-4."
     },
     {
       "flag": "No variance across runs",
       "detail": "No standard deviation or multi-run results are reported; fine-tuning with random initialization and dataset sampling introduces variance that is unmeasured."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "Conala (2018), APPS (2021), and CodeAlpaca (2023) predate the training cutoffs of CodeLlama and CodeGen2; potential test data leakage into model pre-training is never discussed."
     },
     {
       "flag": "ICL baseline potentially weak",
       "detail": "ICL uses randomly selected examples rather than retrieval-based selection; prior work cited by the authors shows retrieval-based ICL significantly outperforms random selection, making PEFT vs ICL comparisons potentially inflated."
     },
     {
       "flag": "Python-only evaluation",
       "detail": "All experiments use Python code generation only, yet the abstract claims PEFT 'superiority and potential over ICL and RAG across a diverse set of LLMs' without qualifying this limitation up front."
     }
   ],
   "cited_papers": [
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "relevance": "Core PEFT technique evaluated; foundational method for parameter-efficient fine-tuning"
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "relevance": "QLoRA technique combining LoRA with quantization; key method evaluated for memory reduction"
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "relevance": "Best-performing LLM family in the study; primary model used for RQ3 and RQ4 analysis"
     },
     {
       "title": "Few-Shot Parameter-Efficient Fine-Tuning is Better and Cheaper than In-Context Learning",
       "relevance": "Prior NLP work showing PEFT advantage; this paper extends those findings to code generation with LLMs"
     },
     {
       "title": "Delta Tuning: A Comprehensive Study of Parameter Efficient Methods for Pre-Trained Language Models",
       "relevance": "Large-scale NLP comparison showing full FT > PEFT; this paper's SE findings contrast with these results"
     },
     {
       "title": "Measuring Coding Challenge Competence With APPS",
       "relevance": "Execution-based benchmark used for RQ4; provides difficulty-stratified evaluation of code generation"
     },
     {
       "title": "CodeT5+: Open Code Large Language Models for Code Understanding and Generation",
       "relevance": "SLM and LLM family evaluated in the study; prior work on code-specific pre-training"
     },
     {
       "title": "Docprompting: Generating Code by Retrieving the Docs",
       "relevance": "RAG baseline approach for code generation; directly compared against PEFT in RQ3"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses the real constraint of single-GPU fine-tuning, with specific memory numbers and code released for practitioners to reproduce."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "PEFT beating full fine-tuning for SLMs contrasts with NLP literature findings, and QLoRA-4bit outperforming LoRA is counterintuitive (lower precision = better)."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns raised; purely a methods comparison paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward empirical comparison with no controversy or competing claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly available at GitHub and all models are open-source; practitioners can reproduce results on a single consumer GPU."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "University of Montreal and Singapore Management University are solid academic institutions but not top-tier AI labs; no industry co-authorship."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "32632312",
         "title": "Exploring the Role of the Cybercrime Underground in the Russia-Ukraine Conflict",
         "points": 4,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=32632312",
         "created_at": "2022-08-28T21:36:55Z"
       },
       {
         "hn_id": "35662520",
         "title": "Learning to Program with Natural Language",
         "points": 3,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=35662520",
         "created_at": "2023-04-22T01:45:40Z"
       },
       {
         "hn_id": "37866902",
         "title": "Getting Bored of Cyberwar",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=37866902",
         "created_at": "2023-10-13T05:03:06Z"
       },
       {
         "hn_id": "37232173",
         "title": "GPT-NER: Named Entity Recognition via Large Language Models",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37232173",
         "created_at": "2023-08-23T05:23:52Z"
       },
       {
         "hn_id": "37168933",
         "title": "Fast as Chita: Neural Network Pruning with Combinatorial Optimization",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37168933",
         "created_at": "2023-08-17T22:16:16Z"
       },
       {
         "hn_id": "35984221",
         "title": "SLiC-HF: Sequence Likelihood Calibration with Human Feedback",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=35984221",
         "created_at": "2023-05-18T04:48:32Z"
       },
       {
         "hn_id": "35263649",
         "title": "A comprehensive capacity analysis of GPT-3 and GPT-3.5 models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=35263649",
         "created_at": "2023-03-22T16:39:00Z"
       },
       {
         "hn_id": "37232871",
         "title": "Vanilla Transformer SOTA for Traffic Forecasting [pdf]",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37232871",
         "created_at": "2023-08-23T07:33:46Z"
       },
       {
         "hn_id": "37958375",
         "title": "Revealing the structure of language model capabilities",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37958375",
         "created_at": "2023-10-20T16:40:14Z"
       },
       {
         "hn_id": "35670419",
         "title": "Fully Autonomous Programming with Large Language Models",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=35670419",
         "created_at": "2023-04-22T20:05:33Z"
       }
     ],
     "top_points": 4,
     "total_points": 22,
     "total_comments": 3
   }
 }