ai-research-survey

scan.json (30493B)

---

 {
   "paper": {
     "title": "MergeRepair: An Exploratory Study on Merging Task-Specific Adapters in Code LLMs for Automated Program Repair",
     "authors": [
       "Meghdad Dehghan",
       "Jie JW Wu",
       "Fatemeh H. Fard",
       "Ali Ouni"
     ],
     "year": 2024,
     "venue": "Empirical Software Engineering (submitted); arXiv",
     "arxiv_id": "2408.09568",
     "doi": "10.48550/arXiv.2408.09568"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Merging task-specific LoRA adapters can improve Automated Program Repair performance by up to 2.38% pass@1 and 4.01% pass@10 on HumanEvalFix without additional training. Performance depends more on which tasks are merged than how many adapters are combined. Non-APR merged adapters can generalize to APR, sometimes outperforming the APR-specific adapter. In continual merging, placing the strongest individual adapter last in the merge sequence yields the best results due to its higher weight influence.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at https://github.com/mqddd/mergerepair (footnote 1, Section 1: 'Our code and replication package are made public')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses publicly available datasets: CommitPackFT from OctoPack [39] and HumanEvalFix benchmark [39]. Both are publicly downloadable."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.8 states the GPU (NVIDIA Tesla V100 32GB) and Table 2 lists hyperparameters, but no Python version, library versions, requirements.txt, or Dockerfile is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is explicitly provided at the GitHub URL. Section 1 states 'Our code and replication package are made public to support open data and open science principles.' Section 8 reaffirms 'We include all scripts used to obtain the results.'"
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-11 are reported as point estimates (e.g., '28.66%') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.7 describes pairwise t-tests with p-values reported at three significance levels (* p<0.1, ** p<0.05, *** p<0.01) throughout Tables 4-11."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cliff's delta effect sizes are reported throughout Tables 4-11 (S for small >=0.11, M for medium >=0.28, L for large >=0.43), as described in Section 3.7."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for the sample size. HumanEvalFix has 165 problems; no discussion of whether this is sufficient for the number of comparisons being made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run point estimates. While 20 samples are generated per problem for pass@k computation, no standard deviation or variance across independent experimental runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Individual task-specific adapters (especially the APR adapter) serve as baselines. Table 3 provides baseline scores for all individual adapters. Equal-weight merging serves as baseline for continual merging (RQ3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models (StarCoder2, Granite, both 2024) and merging methods (TIES-Merging, DARE, weight averaging) are contemporary. The CommitPackFT dataset and HumanEvalFix benchmark are from 2023."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The entire study systematically ablates adapter combinations: testing all subsets of 2, 3, 4, and 5 merged adapters (RQ1-RQ2), and all permutations of merge order (RQ3). The effect of each task and merging method is isolated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: pass@1, pass@10 (reported for all experiments), and RobustPass@1 (RP5@1, reported for selected experiments in Section 5.2, Table 14)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through pass@k execution-based metrics using the bigcode-evaluation-harness library. No human evaluation of generated patches is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Training uses CommitPackFT dataset while evaluation uses the separate HumanEvalFix benchmark (Section 3.7). The instruction-tuned versions of the base models were deliberately avoided because they were trained on the same data (Section 3.5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by task combination, merging method (dare-ties, ties, weight-averaging), and model (StarCoder2, Granite) across all tables. Per-task individual performance is shown in Table 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 shows an example case where individual adapters fail but the merged adapter succeeds (Table 12). Section 5.5 analyzes solved/unsolved problems via Venn diagrams (Figure 11). Section 5.6 tracks problems that became incorrect after merging."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Many merged adapters show degraded performance, reported transparently. For StarCoder2, only 19/45 pass@1 experiments improved (RQ1 summary). T5 (Test & QA) consistently degrades performance. Continual merging with suboptimal order worsens results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '2.38% performance improvement over APR task in terms of pass@1' and '4.01% for pass@10' are confirmed in Tables 4 and 5 (T1-T2-T3 with ties method: 31.04% vs 28.66% = +2.38% pass@1; weight-averaging: 43.12% vs 39.11% = +4.01% pass@10)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims like 'merging adapters can enhance performance' and 'the order of merging affects performance.' The study design uses controlled manipulation (systematic combination of adapters, all permutations of merge order) evaluated on the same benchmark, which is adequate for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Threats to External Validity) explicitly states: 'we conducted experiments only on automatic program repair, along with the other four tasks for Python language... the obtained results are limited to the used tasks and programming language.' The title also bounds scope to 'Code LLMs for Automated Program Repair.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 analyzes Fraction of Sign Difference as an alternative factor explaining performance. Section 7 discusses dataset labeling noise from GPT-4. Section 6.1 notes HumanEvalFix contains diverse samples beyond APR, potentially explaining why non-APR adapters perform well on it."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures pass@k on HumanEvalFix and frames claims in terms of 'performance on the APR benchmark.' Section 3.7 explicitly discusses why pass@k is more reliable than similarity-based metrics (BLEU, CodeBLEU) for measuring functional correctness. Claims match measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: 'starcoder2-3b' and 'granite-3b-code-base' with references to their respective papers [32, 37]. Section 3.5 describes selecting 'base checkpoints (pre-trained only)' of both models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 describes the data structure (commit messages, old contents, new contents) and Table 12 shows one evaluation example, but the actual prompt template used to format instruction-tuning data for the models is not provided. The evaluation prompt template from bigcode-evaluation-harness is not shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports learning rate, scheduler, weight decay, max steps, warmup steps, batch size, gradient accumulation, and LoRA rank for both models. Section 3.8 states temperature=0.2, n=20 samples for evaluation, and lists the specific model layers where LoRA is applied."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach involves fine-tuning LoRA adapters and merging their weight parameters."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes the CommitPackFT dataset origin (filtered from CommitPack by its publishers), task classification via GPT-4 1-shot prompting, and Python split selection. Table 1 shows task distribution across 59,113 samples. Section 3.8 describes perturbation generation for HumanEvalFixPerturbed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides substantive discussion organized into internal, external, construct, and conclusion validity categories."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses specific threats: GPT-4 task labeling may have incorrect labels, experiments limited to two models of the same size (3B), model architecture effects not studied, results limited to Python and APR tasks, docstring perturbation incompatibility with bigcode-evaluation-harness."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (External Validity): 'we conducted experiments only on automatic program repair, along with the other four tasks for Python language... the obtained results are limited to the used tasks and programming language and might not be applicable to other areas or programming languages.' Also acknowledges model size limitation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "CommitPackFT and HumanEvalFix are publicly available datasets. The replication package at GitHub includes scripts to reproduce results. Section 8 states 'Our code and datasets are shared publicly.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes CommitPackFT origin (GitHub repositories via CommitPack, filtered by OctoPack publishers), its structure (commit messages, old/new contents), task classification process (GPT-4 1-shot prompting), and dataset size (59,113 samples across 5 tasks)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (CommitPackFT, HumanEvalFix)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: CommitPackFT → task-specific splits (Table 1 with counts) → LoRA training per task → merging via three methods → evaluation on HumanEvalFix. Figure 1 provides an overview. Section 3.8 describes the perturbation pipeline for robustness evaluation."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: 'This research is supported by a grant from the Natural Sciences and Engineering Research Council of Canada RGPIN-2019-05175, as well as support from computational resources and services provided by Advanced Research Computing at the University of British Columbia.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of British Columbia (Dehghan, Wu, Fard) and École de technologie supérieure (Ouni). No conflict with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSERC is a Canadian government funding agency with no financial stake in the outcomes of this research on adapter merging."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'The authors declare that they have no conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for StarCoder2 or Granite. Section 3.5 describes their training data sources (Stack v2, SWH repositories) but not temporal cutoffs."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEvalFix problems could appear in StarCoder2 or Granite pre-training data. HumanEval was published in 2021 and both models were trained on large code corpora that could include it."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEvalFix is based on HumanEval (published 2021). Both StarCoder2 and Granite were trained after 2021 on large-scale code data. No discussion of whether benchmark problems or their solutions leaked into pre-training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a computational experiment on adapter merging."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or wall-clock time is reported for the main experiments. Section 5.2 mentions evaluation takes 'at least 8-28x time' for perturbed datasets and 'approximately 1 hour' per model for pass@k, but this is incidental and not a systematic cost report."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.8 states experiments used 'one NVIDIA Tesla V100 32GB GPU' but does not report total GPU hours, training time per adapter, or total compute budget for the study."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. Each adapter is trained once and evaluated once (generating 20 samples per problem at temperature 0.2)."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.8 states 'generate 20 samples to calculate pass@1 and pass@10 scores.' For RobustPass@k, '5 different seeds' are used for perturbation generation (Section 5.2). The statistical test uses '400 times' random sampling (Section 3.7)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.8 states 'The hyperparameters are set following the previous works [39, 37]' but no search budget is reported. No indication of how many configurations were tried or whether any tuning was done."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are justified by following configurations from the model developers' papers [39, 37] (Section 3.8). The study reports all adapter combinations rather than cherry-picking the best."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The study performs many pairwise t-tests (hundreds of comparisons across Tables 4-11) but no correction for multiple comparisons (Bonferroni, Holm, etc.) is applied or mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "All adapters (individual and merged) are trained and evaluated by the authors. No discussion of self-comparison bias or independent evaluation is provided."
       },
       "compute_budget_vs_performance": {
         "applies": false,
         "answer": false,
         "justification": "All models are the same size (3B parameters) and merging methods do not require additional training. Compute differences between merging methods are negligible."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.7 argues pass@k is more reliable than similarity metrics, but there is no discussion of whether HumanEvalFix adequately represents real-world APR scenarios. Section 6.1 notes the benchmark 'contained a more diverse set of samples, including instances from tasks beyond APR' but does not discuss construct validity implications."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated directly on benchmark tasks through code generation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. HumanEvalFix is based on HumanEval (2021), and both models were trained on data that could include solutions. CommitPackFT comes from GitHub repositories with no temporal filtering mentioned."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The HumanEvalFix prompt provides the buggy function with test cases, but no analysis of whether this format introduces leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CommitPackFT training examples and HumanEvalFix test problems share structural similarities or come from overlapping code repositories."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used. No canary strings, membership inference, or n-gram overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Merging APR with Improvement and Misc task-adapters achieves up to 2.38% pass@1 improvement and 4.01% pass@10 improvement on StarCoder2.",
       "evidence": "Table 4 shows T1-T2-T3 with ties method achieves 31.04% pass@1 vs 28.66% APR baseline (+2.38%). Table 5 shows T1-T2-T3 with weight-averaging achieves 43.12% pass@10 vs 39.11% (+4.01%).",
       "supported": "strong"
     },
     {
       "claim": "The performance of merged adapters depends more on which task-adapters are merged than on the number of adapters.",
       "evidence": "Across Tables 4-7, no monotonic trend with number of merged adapters. T1-T2-T3 (3 tasks) outperforms T1-T2-T3-T4-T5 (5 tasks) on StarCoder2. The type of task (especially presence of T3/Misc) consistently matters more than count.",
       "supported": "moderate"
     },
     {
       "claim": "Merged adapters from non-APR tasks can generalize to APR, sometimes outperforming the APR-specific adapter.",
       "evidence": "Tables 6-7 (RQ2): T2-T3 achieves 31.95% pass@1 on StarCoder2 with ties (+3.29% over APR). For Granite, 29/33 experiments show improvement over APR adapter (RQ2 summary).",
       "supported": "strong"
     },
     {
       "claim": "In continual merging, placing the best-performing individual adapter last yields the highest overall performance.",
       "evidence": "Tables 8-11 (RQ3): For StarCoder2, T4-T2-T1 (APR last) consistently outperforms other orderings of the same tasks. The pattern is consistent across merging methods and both models.",
       "supported": "strong"
     },
     {
       "claim": "The Misc (T3) task-specific adapter outperforms the APR adapter on the HumanEvalFix benchmark across both models.",
       "evidence": "Table 3: T3 achieves 31.40% vs T1's 28.66% pass@1 on StarCoder2, and 18.63% vs 16.16% on Granite.",
       "supported": "strong"
     },
     {
       "claim": "Continual merging can be more effective than equal-weight merging when merge order is optimized.",
       "evidence": "Section 5.3 and Figures 2-5 show that for every equal-weight merged adapter, there exists a continual merging variant that outperforms it. This is demonstrated across both models and all merging methods.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Small benchmark size",
       "detail": "HumanEvalFix contains only 165 Python problems. With hundreds of pairwise statistical comparisons across Tables 4-11, the statistical power is limited, and no multiple comparison correction is applied."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Both StarCoder2 and Granite were trained on large-scale code data after HumanEval was published (2021). No analysis of whether benchmark problems or solutions leaked into pre-training data."
     },
     {
       "flag": "Marginal practical improvements",
       "detail": "The best improvements (2.38% pass@1, 4.01% pass@10) are small in absolute terms. Many merging configurations degrade performance. On StarCoder2, only 19/45 experiments improved pass@1. The practical significance beyond statistical significance is unclear."
     },
     {
       "flag": "No seed sensitivity analysis",
       "detail": "Each adapter is trained once with one seed. Given that LoRA training and code generation with temperature 0.2 are stochastic, the reported differences may fall within run-to-run variance."
     },
     {
       "flag": "GPT-4-labeled task categories",
       "detail": "The task classification in CommitPackFT was done via 1-shot GPT-4 prompting, introducing potential label noise. The authors acknowledge this threat in Section 7 but do not quantify its impact."
     }
   ],
   "cited_papers": [
     {
       "title": "RepairLlama: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["André Silva", "Sen Fang", "Martin Monperrus"],
       "year": 2023,
       "arxiv_id": "2312.15698",
       "relevance": "Studies LoRA adapters specifically for program repair, directly relevant to adapter-based code LLM fine-tuning for APR."
     },
     {
       "title": "OctoPack: Instruction Tuning Code Large Language Models",
       "authors": ["Niklas Muennighoff", "Qian Liu", "Armel Zebaze"],
       "year": 2023,
       "relevance": "Source of CommitPackFT dataset and HumanEvalFix benchmark used in this study; foundational work on instruction-tuning code LLMs."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The Next Generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "One of the two base code LLMs used in this study; key open-source code generation model."
     },
     {
       "title": "Granite Code Models: A Family of Open Foundation Models for Code Intelligence",
       "authors": ["Mayank Mishra", "Matt Stallone", "Gaoyuan Zhang"],
       "year": 2024,
       "arxiv_id": "2405.04324",
       "relevance": "Second base code LLM used in this study; IBM's open foundation model for code."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["Edward J. Hu", "Yelong Shen", "Phillip Wallis"],
       "year": 2022,
       "relevance": "Foundational adapter method used for all fine-tuning in this study; key parameter-efficient fine-tuning technique."
     },
     {
       "title": "TIES-Merging: Resolving Interference When Merging Models",
       "authors": ["Prateek Yadav", "Derek Tam", "Leshem Choshen"],
       "year": 2024,
       "relevance": "One of three merging techniques evaluated; proposes trim-resolve-merge approach for model parameters."
     },
     {
       "title": "Language Models are Super Mario: Absorbing Abilities from Homologous Models as a Free Lunch",
       "authors": ["Le Yu", "Bowen Yu", "Haiyang Yu"],
       "year": 2023,
       "arxiv_id": "2311.03099",
       "relevance": "Proposes the DARE merging method evaluated in this study; demonstrates parameter dropping and rescaling for model merging."
     },
     {
       "title": "Arcee's MergeKit: A Toolkit for Merging Large Language Models",
       "authors": ["Charles Goddard", "Shamane Siriwardhana", "Malikeh Ehghaghi"],
       "year": 2024,
       "arxiv_id": "2403.13257",
       "relevance": "Toolkit for merging LLMs; demonstrated merged models outperforming constituents on general and medical benchmarks."
     },
     {
       "title": "ReCode: Robustness Evaluation of Code Generation Models",
       "authors": ["Shiqi Wang", "Zheng Li", "Haifeng Qian"],
       "year": 2023,
       "doi": "10.18653/v1/2023.acl-long.773",
       "relevance": "Proposes RobustPass@k metric and perturbation framework adapted in this study for APR robustness evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark (basis of HumanEvalFix used here); foundational work on code generation evaluation."
     },
     {
       "title": "Exploring Parameter-Efficient Fine-Tuning Techniques for Code Generation with Large Language Models",
       "authors": ["Martin Weyssow", "Xin Zhou", "Kisub Kim"],
       "year": 2023,
       "arxiv_id": "2308.10462",
       "relevance": "Empirical study of parameter-efficient fine-tuning for code generation; directly relevant to understanding adapter effectiveness for code tasks."
     },
     {
       "title": "Astraios: Parameter-Efficient Instruction Tuning Code Large Language Models",
       "authors": ["Terry Yue Zhuo", "Armel Zebaze", "Nitchakarn Suppattarachai"],
       "year": 2024,
       "relevance": "Studies parameter-efficient instruction tuning for code LLMs; relevant baseline for understanding adapter fine-tuning in the code domain."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Adapter merging is a practical technique for combining task-specific models without retraining, applicable in resource-constrained settings."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that non-APR merged adapters can match or exceed APR-specific adapters is mildly surprising but not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by this work on adapter merging for program repair."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical study with measured claims."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Replication package released on GitHub with scripts; uses publicly available models and datasets, though setup requires GPU resources."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Academic paper from University of British Columbia and École de technologie supérieure; not from a major AI lab."
     }
   }
 }