ai-research-survey

scan.json (26992B)

---

 {
   "paper": {
     "title": "A Systematic Literature Review of Parameter-Efficient Fine-Tuning for Large Code Models",
     "authors": [
       "Saima Afrin",
       "Md Zahidul Haque",
       "Antonio Mastropaolo"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2504.21569",
     "doi": "10.1145/3796522"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This SLR of 28 papers identifies 19 SE tasks where PEFT has been applied, with Code Summarization (~46.4%) and Code Generation (~35.7%) dominating. Base LoRA is the most widely adopted PEFT technique (29 task instances), followed by Adapter-based (23) and Prompt Tuning (14). About 72% of generative and 62% of non-generative task evaluations report performance on par with or better than full fine-tuning, while ~29% of reviewed works lack any direct comparison to full fine-tuning baselines.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a replication package at https://github.com/alvi75/SLR-PEFT containing 'search strings, extracted datasets, and study classifications' (Section 8), but no analysis code or scripts are described as being released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'we have made our search strings, extracted datasets, and study classifications publicly available in our replication package' at a GitHub URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Publish or Perish (PoP) software for Google Scholar queries but provides no version information or other tool specifications needed to replicate the analysis environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 provides detailed methodology: search strings (with examples and full set in replication package), 4 databases queried, time frame (2019-2025), explicit exclusion criteria, venue list, filtering pipeline with counts (1,146→26→25→28), and snowballing procedure. A competent researcher could follow this to reproduce the SLR."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not run experiments or compute statistical estimates requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is an SLR that synthesizes findings qualitatively and through vote-counting, not through statistical testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The SLR does not perform meta-analysis with effect size computation; it reports categorical outcomes (improvement/decline/parity) from individual studies."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable to a systematic literature review — the corpus size (28 papers) is determined by the search and filtering process, not by statistical power considerations."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "The SLR does not run experiments and thus has no experimental variance to report."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions prior PEFT surveys (Han et al. [43], Lialin et al. [76]) in Section 2.3 but does not formally compare its findings, scope, or coverage against these prior reviews."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No formal baseline comparison against prior surveys is conducted, so contemporaneity of baselines cannot be assessed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "A systematic literature review has no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "A systematic literature review does not measure performance with evaluation metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of outputs is not applicable to a literature review."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "A systematic literature review does not use train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are extensively broken down by generative vs. non-generative tasks (Table 1), by PEFT method (Table 2, Fig. 5), by model architecture (Section 4.2.1), and by individual SE task (Figs. 3-4). Table 6 provides per-task performance and efficiency breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 reports cases where PEFT showed performance degradation (↓ symbols). Section 4.3 discusses specific failures: 'Code Translation may be less amenable to parameter-efficient strategies,' Adapter and MAM methods show declines in Code Translation, and Pass-Tuning shows performance decline in Code Generation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 6 explicitly reports negative outcomes where PEFT underperformed full fine-tuning. Section 4.3 notes that '~29% of the reviewed works do not include any direct comparison with full fine-tuning' and identifies specific cases of performance degradation."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: they do examine PEFT across SE tasks (Section 4.1), analyze 28 papers (Sections 3-4), identify configuration patterns (Tables 3-5), and produce a taxonomy (Fig. 3). The claim of 'comprehensive taxonomy' is supported by the detailed categorization."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal-style claims such as 'PEFT techniques can make the training and deployment of Large Code Models more practical and sustainable' and 'PEFT is not just a workaround for scarce resources, but a robust optimization paradigm' (Section 9). These conclusions are based on vote-counting (counting papers reporting improvement) without formal meta-analysis, effect size aggregation, or controlling for publication bias in the underlying studies."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly bounds its scope: 28 papers from 2019-2025, from specified top-tier venues (Section 3.1), with explicit venue list and exclusion criteria. The title accurately reflects the content as an SLR. Section 7 acknowledges potential gaps from venue restrictions."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its synthesis findings. For example, the finding that ~72% of studies report PEFT improvements could be driven by publication bias (positive results are more publishable), selection bias from venue filtering, or authors' tendency to report their best configurations. None of these alternatives are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures 'number of studies reporting improvement/decline' and frames this as evidence of PEFT effectiveness. The gap between 'papers report improvement' (proxy) and 'PEFT is actually effective in practice' (claimed outcome) is not acknowledged. Vote-counting is known to be a weak evidence synthesis method that ignores effect sizes, study quality, and sample sizes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The SLR does not use any AI models itself; it reviews papers that use models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The SLR does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "The SLR does not run experiments with hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this SLR."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full pipeline: search strings with examples (Section 3.1), 4 databases queried, 1,146 initial results → relevance screening → venue-based filtering (16 venues listed) → exclusion criteria (3 explicit rules) → 26 papers → manual analysis → 25 → snowballing → 28 final papers. Figure 1 visualizes the pipeline with counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides a dedicated discussion covering construct validity, internal validity, and external validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: risk of task/method misclassification mitigated by multiple-author review, inconsistencies from 'limited methodological details in some primary studies, particularly regarding adapter-based or hybrid PEFT configurations,' and potential missed studies 'due to terminology variation, indexing limitations, or query constraints in platforms like Google Scholar.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states scope boundaries: time frame 2019-2025, 16 specific venues, exclusion of preprints (with 2 exceptions), 28-paper corpus. Section 7 acknowledges 'some relevant studies may have been missed' and 'there is a possibility of selection bias and under-representation of work published in non-ranked venues.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'we have made our search strings, extracted datasets, and study classifications publicly available in our replication package [1]' at https://github.com/alvi75/SLR-PEFT."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes data collection in detail: 4 databases (IEEE Xplore, ACM Digital Library, Springer Link, Google Scholar via PoP), tailored search strings with examples provided, time frame 2019-2025, and the split into 7 queries for Google Scholar due to its 256-character limit."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper selection process (analogous to recruitment) is thoroughly described: search strings in Section 3.1, venue selection informed by CSRankings plus additional SE venues, and explicit exclusion criteria in Section 3.2.1. Two reviewers independently assessed papers with disagreements resolved through consensus."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the complete pipeline: 1,146 initial papers → search string application (A) → exclusion criteria (B) → 26 papers → manual analysis (C) → 25 papers → snowballing (D) → 28 final papers (E). Sections 3.1 and 3.2 describe each stage with counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: all three authors are from William & Mary, USA, with email addresses provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funding cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this systematic literature review."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method to cost."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows Kitchenham et al.'s SLR guidelines (Section 3). It uses structured search strings across 4 databases, defined inclusion/exclusion criteria, a multi-stage filtering pipeline documented in Figure 1, and backward snowballing. The methodology is systematic and reproducible."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The SLR filters by venue quality (CSRankings-based selection) but does not assess the methodological quality of individual included studies. All 28 papers are treated equally regardless of their internal validity, sample sizes, or experimental rigor. No quality scoring rubric or risk-of-bias assessment is applied."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. By restricting to top-tier venues and excluding preprints, the sample likely overrepresents positive PEFT results. Section 7 discusses selection bias from venue filtering but never addresses the possibility that published papers systematically skew toward positive outcomes for PEFT methods."
       }
     }
   },
   "claims": [
     {
       "claim": "PEFT techniques have been applied to 19 diverse SE tasks, with Code Summarization (~46.4%) and Code Generation (~35.7%) being the most frequently explored.",
       "evidence": "Table 1 lists 19 SE tasks with corresponding papers. Section 4.1 reports the frequency analysis. Figure 4 shows trends over time.",
       "supported": "strong"
     },
     {
       "claim": "Encoder-decoder architectures are the most prevalent for PEFT optimization, appearing in 29 SE task instances across 12 studies.",
       "evidence": "Section 4.2.1 reports architecture distribution: encoder-decoder (29 instances/12 studies), encoder-only (22/10), decoder-only (19/12). CodeT5 is the most frequently adapted model (19 instances/8 studies).",
       "supported": "strong"
     },
     {
       "claim": "Base LoRA is the most widely adopted PEFT technique, appearing in 29 SE task instances.",
       "evidence": "Section 4.2.2 and Table 2 document LoRA usage across tasks. Figure 5 shows the heatmap of PEFT method distribution.",
       "supported": "strong"
     },
     {
       "claim": "~72% of generative task entries report performance improvements over full fine-tuning, and ~54% also report efficiency gains.",
       "evidence": "Section 4.3 and Table 6 present performance/efficiency outcomes using symbolic indicators (↑/↓/→) for 34 generative task entries.",
       "supported": "moderate"
     },
     {
       "claim": "~62% of non-generative task entries report performance improvements, with ~50% showing efficiency gains.",
       "evidence": "Section 4.3 and Table 6 present outcomes for 16 non-generative task entries.",
       "supported": "moderate"
     },
     {
       "claim": "~29% of reviewed works do not include any direct comparison with full fine-tuning baselines.",
       "evidence": "Section 4.3 notes this gap: 'This selection criterion led to the inclusion of 20 studies in our evaluation' out of 28 total.",
       "supported": "strong"
     },
     {
       "claim": "Decoder-only models appeared exclusively in generative tasks.",
       "evidence": "Section 4.2.1 documents that decoder-only models were used only in NL2Code, Code2NL, and Code2Code tasks — all generative.",
       "supported": "strong"
     },
     {
       "claim": "PEFT is not just a workaround for scarce resources, but a robust optimization paradigm for LCMs.",
       "evidence": "Section 9 makes this conclusion based on the synthesis of 28 studies. Evidence is the aggregate of Table 6 outcomes showing majority of entries report improvement or parity.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of included studies",
       "detail": "The SLR treats all 28 papers equally without assessing their individual methodological quality, sample sizes, or experimental rigor. A study with questionable methodology is given the same weight as a rigorous multi-dataset evaluation. This is a well-known limitation that can launder weak results through uncritical aggregation."
     },
     {
       "flag": "Vote-counting as evidence synthesis",
       "detail": "The paper's main quantitative finding (~72% improvement) is based on vote-counting — tallying how many studies report improvement vs. decline. This is the weakest form of evidence synthesis. It ignores effect sizes, confidence intervals, study quality, and sample sizes. A study showing 0.1% improvement counts the same as one showing 20% improvement."
     },
     {
       "flag": "Publication bias not addressed",
       "detail": "By restricting to top-tier venues and excluding preprints (with only 2 exceptions), the corpus likely overrepresents positive findings about PEFT. Studies finding that PEFT does not work are less likely to be published in top venues. This systematic bias is not discussed in Section 7 (Threats to Validity)."
     },
     {
       "flag": "Small corpus for broad conclusions",
       "detail": "28 papers is a small corpus from which to draw general conclusions about PEFT effectiveness across 19 SE tasks. Several tasks have only 1-2 supporting papers (e.g., Code Review Generation: 1, Method Name Recommendation: 1, Protocol Buffer Transformation: 1), yet findings are presented as characterizing the task."
     }
   ],
   "cited_papers": [
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["Edward J. Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2022,
       "relevance": "Foundational PEFT technique (Low-Rank Adaptation) that is the most widely adopted method in the reviewed SE literature."
     },
     {
       "title": "QLoRA: Efficient Finetuning of Quantized LLMs",
       "authors": ["Tim Dettmers", "Artidoro Pagnoni", "Ari Holtzman", "Luke Zettlemoyer"],
       "year": 2023,
       "arxiv_id": "2305.14314",
       "relevance": "Quantized LoRA variant enabling efficient fine-tuning of large models, applied to code summarization and generation tasks."
     },
     {
       "title": "A systematic literature review on the use of deep learning in software engineering research",
       "authors": ["Cody Watson", "Nathan Cooper", "David Nader Palacio"],
       "year": 2022,
       "relevance": "Provides the SE task taxonomy used as the basis for this SLR's search strategy and task classification."
     },
     {
       "title": "Exploring parameter-efficient fine-tuning techniques for code generation with large language models",
       "authors": ["Martin Weyssow", "Xin Zhou", "Kisub Kim"],
       "year": 2023,
       "relevance": "Empirical study of PEFT techniques for code generation, one of the 28 primary studies in this SLR."
     },
     {
       "title": "An Empirical Study of Parameter-Efficient Fine-Tuning Methods for Pre-Trained Code Models",
       "authors": ["Jiaxing Liu", "Chaofeng Sha", "Xin Peng"],
       "year": 2023,
       "doi": "10.1109/ASE56229.2023.00125",
       "relevance": "Comprehensive empirical evaluation comparing multiple PEFT methods across SE tasks, one of the key primary studies."
     },
     {
       "title": "Comprehensive Fine-Tuning Large Language Models of Code for Automated Program Repair",
       "authors": ["Kai Huang", "Jian Zhang", "Xinlei Bao"],
       "year": 2025,
       "doi": "10.1109/TSE.2025.3532759",
       "relevance": "Evaluates LoRA and other PEFT methods for automated program repair, demonstrating PEFT effectiveness in code repair tasks."
     },
     {
       "title": "Beyond PEFT: Layer-Wise Optimization for More Effective and Efficient Large Code Model Tuning",
       "authors": ["Chaozheng Wang", "Jia Feng", "Shuzheng Gao"],
       "year": 2025,
       "relevance": "Proposes going beyond standard PEFT with layer-wise optimization, revealing PEFT limitations on private codebases vs. public datasets."
     },
     {
       "title": "Pass-Tuning: Towards Structure-Aware Parameter-Efficient Tuning for Code Representation Learning",
       "authors": ["Nuo Chen", "Qiushi Sun", "Jianing Wang"],
       "year": 2023,
       "doi": "10.18653/v1/2023.findings-emnlp.42",
       "relevance": "Introduces a structure-aware PEFT method leveraging AST information for code representation learning."
     },
     {
       "title": "No more fine-tuning? an experimental evaluation of prompt tuning in code intelligence",
       "authors": ["Chaozheng Wang", "Yuanhang Yang", "Cuiyun Gao"],
       "year": 2022,
       "doi": "10.1145/3540250.3549113",
       "relevance": "Evaluates prompt tuning across multiple code intelligence tasks, comparing hard and soft prompts for SE applications."
     },
     {
       "title": "Parameter-efficient fine-tuning for large models: A comprehensive survey",
       "authors": ["Zeyu Han", "Chao Gao", "Jinyang Liu"],
       "year": 2024,
       "relevance": "General PEFT survey across NLP domains that this SLR extends specifically to software engineering tasks."
     },
     {
       "title": "MFTCoder: Boosting Code LLMs with Multitask Fine-Tuning",
       "authors": ["Bingchang Liu", "Chaoyu Chen", "Zi Gong"],
       "year": 2024,
       "doi": "10.1145/3637528.3671609",
       "relevance": "Applies LoRA and QLoRA for multitask fine-tuning of code LLMs across code generation, completion, and test generation."
     },
     {
       "title": "Resource-Efficient & Effective Code Summarization",
       "authors": ["Saima Afrin", "Joseph Call", "Khai-Nguyen Nguyen"],
       "year": 2025,
       "arxiv_id": "2502.03617",
       "relevance": "Demonstrates QLoRA effectiveness for code summarization with large models up to 33B parameters."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners can use the taxonomy and Tables 3-6 to select PEFT methods for their own fine-tuning tasks, but no tool or code is provided."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected findings — PEFT works well and is increasingly popular in SE, with no surprising or contrarian conclusions."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns are raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict with prior work; positions itself as filling a gap rather than challenging existing findings."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Replication package with data artifacts is available on GitHub, but there is nothing to run or demo."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from William & Mary, an academic institution without high brand recognition in AI/LLM research circles."
     }
   }
 }