ai-research-survey

scan.json (33195B)

---

 {
   "paper": {
     "title": "The Impact of Fine-tuning Large Language Models on Automated Program Repair",
     "authors": [
       "Roman Machácek",
       "Anastasiia Grishina",
       "Max Hort",
       "Leon Moonen"
     ],
     "year": 2025,
     "venue": "41st IEEE International Conference on Software Maintenance and Evolution (ICSME 2025)",
     "arxiv_id": "2507.19909",
     "doi": "10.1109/ICSME64153.2025.00042"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Full fine-tuning of LLMs on bug-fix pairs improves poorly-performing models (CodeT5, Bloom) but degrades the best-performing ones (DeepSeekCoder, StarCoder) due to data distribution mismatch and overfitting. Parameter-efficient fine-tuning with LoRA consistently outperforms both full fine-tuning and IA3, achieving up to 225% improvement over base models while using less than 1% of trainable parameters. LoRA hyperparameters (rank and scaling factor) have negligible impact on performance, suggesting default values are sufficient.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package with code and results is provided via Zenodo (doi:10.5281/zenodo.16359186), referenced in Section VI and footnote 1. The authors state their code 'builds on the code provided by Jiang et al., adapting it to accommodate the benchmarking of additional models and parameter-efficient fine-tuning.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three benchmarks (QuixBugs, Defects4J, HumanEval-Java) are publicly available. The CLM fine-tuning dataset is available via the clm GitHub repository. The Zenodo replication package includes results data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using A100 and V100 GPUs and the Hugging Face library for models and PEFT, but does not specify library versions, Python version, or provide a requirements.txt or Dockerfile. Notably, the authors criticize Jiang et al. for not specifying the Java version used, yet do not specify their own Java version either."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package (doi:10.5281/zenodo.16359186) is explicitly described as enabling 'replication and verification of our work' (Section VI). The paper provides detailed preprocessing in Listing 1 and describes the experimental setup thoroughly in Section III."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (counts of plausible patches, single CodeBLEU/exact-match values) with no confidence intervals or error bars anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'LoRA performs better than IA3 in 21 out of 24 cases') based solely on comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline context for improvements: 'by using LoRA for CodeGen-2B, we used only 0.09% of trainable parameters of the full model, while achieving performance gains of 172%, 225%, 153% on the QuixBugs, HumanEval-Java and Defects4J benchmarks.' Tables consistently show base vs. fine-tuned numbers for comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for generating exactly 10 patches per problem (only cited as convention from related works), nor for the selection of benchmark sizes. The choice of models and benchmarks follows prior work without independent justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single experimental runs with no variance, standard deviation, or spread measures reported. The paper does not state whether experiments were repeated or report any measure of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Each RQ uses the non-fine-tuned base models as baselines (Table I). RQ3 compares PEFT against both base and full fine-tuning (Table V). Results are also compared with Jiang et al. [8] and Li et al. [58]."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Jiang et al. (ICSE 2023) and Li et al. (ASE 2024), which are contemporary works. The models used (DeepSeekCoder, CodeLlama2, StarCoder) are recent code LLMs."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 systematically varies LoRA hyperparameters (rank and scaling factor) across 7 values each to measure their individual contribution. The study also compares with/without buggy lines (RQ1), and base vs. full FT vs. LoRA vs. IA3 (RQ2-3), which constitute controlled ablations of the training regimen."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses plausible patch count, CodeBLEU, exact match, and training/validation loss as evaluation metrics (Section III-C and Table II)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated through test suite execution, CodeBLEU, and exact match. The authors acknowledge this limitation: 'plausibility shows whether a patch passes all available tests but is not a guarantee of its correctness' and note that 'manual correctness verification is prone to subjectivity' (Section IV-E)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The three APR benchmarks (QuixBugs, HumanEval-Java, Defects4J) are strictly held-out from the CLM fine-tuning dataset. Defects4J-related patches were explicitly removed from CLM using AST comparison (Section III-A). The CLM dataset itself is split 80/20 for training/validation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (6 families, 15 model sizes), per benchmark (QuixBugs, HumanEval-Java, Defects4J), per training method (base, full FT, LoRA, IA3), and per epoch (Tables I-V). Patch outcome categories (Plausible, Timeout, Uncompilable, Wrong, Unknown) are also defined."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "A major finding is that full fine-tuning degrades best-performing models (RQ2): 'DeepSeekCoder and StarCoder show deterioration after fine-tuning compared to using the base models.' The paper discusses causes including overfitting and data distribution mismatch."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper prominently reports that full fine-tuning worsens performance of strong models (Table III). IA3 underperforms LoRA contrary to expectations from prior work. Including buggy lines fails to improve performance in 29/45 cases. These are substantive negative findings."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that full fine-tuning decreases performance due to data distributions and overfitting, and that PEFT achieves better results. Both claims are supported by Tables III and V respectively, with detailed discussion in RQ2 and RQ3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The study uses controlled single-variable manipulation: same models, same benchmarks, same datasets, varying only the training regimen (none, full FT, LoRA, IA3). This is adequate for claims that training method affects performance. The causal attribution to 'overfitting' is supported by training/validation loss divergence in Table II."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'The Impact of Fine-tuning Large Language Models on Automated Program Repair' and abstract speak broadly about 'LLMs' and 'APR' without bounding to the tested setting: Java only, 6 specific model families, 3 specific benchmarks, and one fine-tuning dataset (CLM). The conclusions do not explicitly restrict generalization scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E discusses multiple alternative explanations: data leakage from pre-training, data distribution mismatch between CLM and benchmarks, Java version effects, library versioning issues, benchmark non-representativeness, and plausibility vs. correctness concerns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges that plausibility is a proxy for correctness: 'Plausibility shows whether a patch passes all available tests but is not a guarantee of its correctness. One way to assess the correctness of patches is by manually checking' (Section IV-E)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names and sizes are provided: CodeGen-Multi 350M/2B/6B, CodeT5-small (60M)/base (220M)/large (770M), StarCoderBase 1B/3B, DeepSeekCoder-Base 1.3B/6.7B (v1), Bloom 560M/1B7/7B1, CodeLlama2 7B. DeepSeekCoder version is explicitly stated as 'v1' (Section III-B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 1 provides the complete input preprocessing format for all six model families, showing the actual prompt structure with concrete code examples including special tokens (<FILL_ME>, <|fim_begin|>, <fim_prefix>, etc.)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "LoRA hyperparameters are reported (r=8, α=16 defaults) and RQ4 systematically varies them. However, core training hyperparameters for full fine-tuning (learning rate, batch size, optimizer, weight decay) are not reported. Inference parameters (temperature, sampling strategy) for generating the 10 patches are also absent."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models directly generate patches from code input without any multi-step reasoning, tool use, or agent framework."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Listing 1 documents model-specific preprocessing. Section III-A describes CLM dataset construction: 1,083,185 commits filtered to single-hunk patches, Defects4J patches removed via AST comparison, resulting in 143,666 instances split 80/20. Two input scenarios (with/without buggy line) are described in Section III-D."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E 'Threats to Validity' provides a dedicated multi-paragraph discussion of limitations covering benchmark representativeness, data leakage, library versioning, dataset distribution mismatch, and evaluation methodology."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: 'benchmarks like HumanEval-Java and QuixBugs were created from simple projects and consist of bugs that are not representative of complex real-world bugs,' 'versioning of various libraries caused by internal dependencies leads to internal problems of benchmarks,' and specific discussion of data leakage risks for the models used."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses what might limit its results (benchmark representativeness, data leakage) but does not explicitly state what the results do NOT show or which settings/claims are excluded. The title and conclusions speak broadly about 'LLMs' and 'APR' without bounding conclusions to Java, the specific model families tested, or the specific benchmarks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Zenodo replication package (doi:10.5281/zenodo.16359186) contains 'code and results.' All benchmarks are publicly available (Defects4J, QuixBugs, HumanEval-Java) and the CLM dataset is available via the clm GitHub repository."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes all datasets: Defects4J 2.0.1 with 835 active bugs (219 used), QuixBugs with 40 Java programs, HumanEval-Java with 163 bugs manually created from HumanEval, and CLM dataset from 1,083,185 GitHub commits (March 2011–2018) filtered to 143,666 instances."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard public benchmarks and a publicly available fine-tuning dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The CLM pipeline is documented: 1,083,185 commits → filtered to single-hunk statement updates/insertions → Defects4J patches removed via AST comparison → 143,666 instances → 129,300 training (80%) / 14,366 test (20%). Model-specific preprocessing is documented in Listing 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section discloses: Research Council of Norway secureIT project (IKTPLUSS #288787), EU Horizon Europe Marie Skłodowska-Curie Actions (#101151798), and eX3 infrastructure (contract #270053)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: University of Bern and Simula Research Laboratory. None of the authors are affiliated with the companies that created the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (Research Council of Norway, EU Horizon Europe) are public research funding bodies with no commercial interest in which fine-tuning method performs best."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the six model families (CodeGen, CodeT5, StarCoder, DeepSeekCoder, Bloom, CodeLlama2), despite acknowledging that data leakage from pre-training is a concern."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses overlap at two levels: (1) for fine-tuning, 'all patches related to projects in Defects4J were removed based on a comparison using abstract syntax trees'; (2) for pre-training, 'pre-training of models uses large datasets, usually mined from GitHub. This may cause data leakage, where the benchmarks may have been seen by models at some point during pre-training.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval-Java was specifically created to 'eliminate the threat that models have already seen the dataset during pre-training.' The paper acknowledges contamination risk for other benchmarks and notes that 'None of the models achieved close to the 100% benchmark performance, making it clear that the effect of data leakage is not the only factor.' The CLM dataset was decontaminated of Defects4J entries."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study of LLM fine-tuning methods."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on public code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for generating the 10 patches per problem across 15 models and 3 benchmarks."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'A100, and V100 GPUs' and that hardware limitations prevented fine-tuning some larger models, but does not quantify total GPU hours, training time, or energy consumption for any experiment."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds, seed sensitivity, or results across multiple seeds anywhere in the paper. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'we let them generate 10 patches for each problem, as done in related works [8, 53, 56–58]' (Section III-C). This defines the evaluation protocol."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "For the main experiments (RQ1-3), core training hyperparameters (learning rate, batch size, optimizer) are not reported, so it is unknown whether any search was performed. LoRA defaults were taken from Hugging Face. RQ4 reports a systematic sensitivity analysis but this is separate from the main results."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "For LoRA and IA3, the paper explicitly states they used 'default parameters recommended by Hugging Face PEFT' (r=8, α=16). For RQ4, all configurations are reported (Figures 3-6), not just the best. For RQ3, they chose epoch 1 based on validation metric stagnation observed in Table IV."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own pipeline. They compare with Jiang et al. and Li et al. but note result discrepancies without discussing author-evaluation bias or the possibility that their implementation of baselines may underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table VI reports trainable parameter counts for PEFT methods, but performance is not analyzed as a function of compute budget. Full fine-tuning and PEFT are compared on outcomes without normalizing for or reporting their compute costs."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E explicitly discusses construct validity: 'benchmarks like HumanEval-Java and QuixBugs were created from simple projects and consist of bugs that are not representative of complex real-world bugs,' and plausibility vs. correctness is acknowledged as a measurement limitation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models directly generate patches from code input without any agent framework or multi-step pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that pre-training datasets may include benchmark solutions since benchmarks are public GitHub repositories. HumanEval-Java was specifically created to mitigate temporal leakage: 'The reason for doing so is to eliminate the threat that models have already seen the dataset during pre-training.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 systematically tests whether including buggy lines (additional information that may not be available in real-world scenarios) affects performance. The paper analyzes 'how including or omitting the buggy line(s) affects the LLM's performance,' which is a form of feature leakage analysis."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The CLM fine-tuning dataset was explicitly decontaminated: 'all patches related to projects in Defects4J were removed based on a comparison using abstract syntax trees' (Section III-A) to ensure independence between fine-tuning and benchmark data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "AST-based comparison was used as a concrete decontamination method to remove Defects4J-related patches from the CLM fine-tuning dataset: 'all patches related to projects in Defects4J were removed based on a comparison using abstract syntax trees' (Section III-A, citing Zhu et al.)."
       }
     }
   },
   "claims": [
     {
       "claim": "Larger models generally perform better on APR benchmarks without fine-tuning (in 34 out of 48 cases).",
       "evidence": "Table I shows per-model, per-benchmark results. For example, CodeGen-6B outperforms CodeGen-350M in all cases, and DeepSeekCoder-6.7b achieves the best results (33/40 QuixBugs, 107/163 HumanEval-Java, 89/219 Defects4J). Section IV-A summarizes the trend.",
       "supported": "strong"
     },
     {
       "claim": "Full fine-tuning improves poorly-performing models (CodeT5, Bloom) but degrades the best-performing models (DeepSeekCoder, StarCoder).",
       "evidence": "Table III: CodeT5-base improves from 0/4/8 to 17/39/75 (QB/HE/D4J), while DeepSeekCoder-1.3b drops from 33/94/72 to 15/64/80. StarCoder-3b drops from 32/94/63 to 11/37/63 after 3 epochs of full fine-tuning.",
       "supported": "strong"
     },
     {
       "claim": "PEFT with LoRA outperforms full fine-tuning and IA3 in most cases, achieving up to 225% improvement over base models while using <1% of trainable parameters.",
       "evidence": "Table V: LoRA outperforms IA3 in 21/24 cases. CodeGen-2B with LoRA achieves 19/81/98 (QB/HE/D4J) vs. base 13/44/20, using only 0.09% of parameters (Table VI). Section IV-C provides detailed analysis.",
       "supported": "strong"
     },
     {
       "claim": "LoRA hyperparameters (rank and scaling factor) have negligible impact on APR performance.",
       "evidence": "Figures 3-6 show CodeBLEU varies only between 0.60-0.64 and exact match shows moderate variation across 7 values of rank and scaling factor for CodeGen-2B. Section IV-D concludes 'either rank or scaling factor does not affect the performance of models significantly.'",
       "supported": "moderate"
     },
     {
       "claim": "Including buggy lines in the prompt does not consistently improve APR performance.",
       "evidence": "Table I: adding buggy lines fails to improve performance in 29 out of 45 cases. It helps for the best-performing models on Defects4J (all six model sizes improved) but rarely helps on QuixBugs.",
       "supported": "strong"
     },
     {
       "claim": "PEFT-trained models reach performance plateau after the first epoch, with metrics stagnating.",
       "evidence": "Table IV shows that CodeBLEU, loss, and exact match stagnate after epoch 1 for nearly all models under both LoRA and IA3. This motivated selecting epoch 1 for the benchmark evaluation in Table V.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across all RQs are from single experimental runs with no error bars, confidence intervals, standard deviations, or repeated trials. Given that LLM generation is stochastic, results could vary across runs. This is particularly concerning for close comparisons."
     },
     {
       "flag": "Missing core training hyperparameters",
       "detail": "Learning rate, batch size, optimizer, weight decay, and warmup schedule are not reported for full fine-tuning experiments. Inference parameters (temperature, sampling strategy) for generating the 10 patches are also absent. These can significantly affect results."
     },
     {
       "flag": "Hyperparameter sensitivity tested on only one model",
       "detail": "RQ4 systematically varies LoRA hyperparameters but only for CodeGen-2B. The conclusion that 'rank or scaling factor does not affect the performance of models significantly' is generalized from a single model, which may not hold for other architectures."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims ('LoRA performs better than IA3 in 21 out of 24 cases') are based purely on comparing raw numbers. Without significance testing, some observed differences may be within noise margins, especially given the absence of variance reporting."
     }
   ],
   "cited_papers": [
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00125",
       "relevance": "Core prior work evaluating code LLMs for APR with full fine-tuning; framework and dataset used in this study."
     },
     {
       "title": "Exploring Parameter-Efficient Fine-Tuning of Large Language Model on Automated Program Repair",
       "authors": ["G. Li", "C. Zhi", "J. Chen", "J. Han", "S. Deng"],
       "year": 2024,
       "doi": "10.1145/3691620.3695066",
       "relevance": "Closest related work investigating PEFT techniques for APR; results compared directly with this study."
     },
     {
       "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["A. Silva", "S. Fang", "M. Monperrus"],
       "year": 2024,
       "arxiv_id": "2312.15698",
       "relevance": "Demonstrated LoRA-based fine-tuning of CodeLlama-7b for APR, finding LoRA outperformed full fine-tuning."
     },
     {
       "title": "Comprehensive Fine-Tuning Large Language Models of Code for Automated Program Repair",
       "authors": ["K. Huang", "J. Zhang", "X. Bao", "X. Wang", "Y. Liu"],
       "year": 2025,
       "doi": "10.1109/TSE.2025.3532759",
       "relevance": "Concurrent work studying fine-tuning of code LLMs for APR."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00129",
       "relevance": "Studied LLMs for APR without fine-tuning, showing promising zero-shot repair results."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis", "Z. Allen-Zhu", "Y. Li", "S. Wang", "L. Wang", "W. Chen"],
       "year": 2022,
       "relevance": "Foundational PEFT method used extensively in this study; core technique for parameter-efficient fine-tuning of LLMs."
     },
     {
       "title": "Exploring Parameter-Efficient Fine-Tuning Techniques for Code Generation with Large Language Models",
       "authors": ["M. Weyssow", "X. Zhou", "K. Kim", "D. Lo", "H. Sahraoui"],
       "year": 2025,
       "doi": "10.1145/3714461",
       "relevance": "Investigated PEFT for code generation, found PEFT outperforms ICL and RAG across diverse LLMs."
     },
     {
       "title": "Astraios: Parameter-Efficient Instruction Tuning Code Large Language Models",
       "authors": ["T. Y. Zhuo", "A. Zebaze", "N. Suppattarachai", "L. von Werra", "H. de Vries", "Q. Liu", "N. Muennighoff"],
       "year": 2024,
       "arxiv_id": "2401.00788",
       "relevance": "One of the first studies of PEFT for code LLMs, found LoRA offers the most competitive cost-performance trade-offs for 16B models."
     },
     {
       "title": "A Systematic Literature Review of Parameter-Efficient Fine-Tuning for Large Code Models",
       "authors": ["M. Z. Haque", "S. Afrin", "A. Mastropaolo"],
       "year": 2025,
       "arxiv_id": "2504.21569",
       "relevance": "Recent systematic review of PEFT techniques for code LLMs, providing broader context for this study."
     },
     {
       "title": "Multi-Objective Fine-Tuning for Enhanced Program Repair with LLMs",
       "authors": ["B. Yang", "H. Tian", "J. Ren", "H. Zhang", "J. Klein", "T. F. Bissyandé", "C. L. Goues", "S. Jin"],
       "year": 2024,
       "arxiv_id": "2404.12636",
       "relevance": "Investigated multi-objective fine-tuning approaches for LLM-based APR using PEFT."
     },
     {
       "title": "Automated Repair of Programs from Large Language Models",
       "authors": ["Z. Fan", "X. Gao", "M. Mirchev", "A. Roychoudhury", "S. Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00128",
       "relevance": "Studied automated program repair using LLMs, part of the growing body of work on LLM-based APR."
     },
     {
       "title": "Few-Shot Parameter-Efficient Fine-Tuning Is Better and Cheaper than in-Context Learning",
       "authors": ["H. Liu", "D. Tam", "M. Muqeeth", "J. Mohta", "T. Huang", "M. Bansal", "C. A. Raffel"],
       "year": 2022,
       "relevance": "Introduced the IA3 adapter method used in this study; demonstrated PEFT advantages over ICL."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners fine-tuning LLMs for code repair can directly use the finding that LoRA with default settings outperforms full fine-tuning at a fraction of the compute cost."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that full fine-tuning degrades performance of strong models is somewhat surprising but has been observed in adjacent domains."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications in the findings."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; straightforward empirical comparison."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Replication package available on Zenodo but it is a research reproduction setup, not a ready-to-use tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses recognizable models (DeepSeekCoder, CodeLlama, StarCoder) but authors are from academic institutions (Simula, Univ of Bern)."
     }
   }
 }