ai-research-survey

scan.json (30990B)

---

 {
   "paper": {
     "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
     "authors": ["André Silva", "Sen Fang", "Martin Monperrus"],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2312.15698",
     "doi": "10.1109/TSE.2025.3581062"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RepairLLaMA demonstrates that parameter-efficient fine-tuning (LoRA) with repair-specific code representations incorporating fault localization signals outperforms both full fine-tuning and zero-shot GPT-3.5/GPT-4 on three Java program repair benchmarks. The best representation (IR4xOR2) combines buggy code as comments with an infilling mask, correctly fixing 144 Defects4J v2, 109 HumanEval-Java, and 20 GitBug-Java bugs. LoRA adapters with only 4M parameters (1600x smaller than the base LLM) outperform full-parameter fine-tuning on CodeLLaMA-7B, suggesting regularization benefits. Representations without fault localization perform substantially worse, confirming that task-specific input signals are essential for fine-tuning effectiveness.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 'we publish our source code, models, and artifacts at https://github.com/ASSERT-KTH/repairllama' (Section I, contributions). A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All three evaluation benchmarks (Defects4J, HumanEval-Java, GitBug-Java) are publicly available. The fine-tuning dataset Megadiff is also public. The paper additionally references the GitHub repository for supplementary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (4xA100 40GB GPUs for training, single A100 for inference, Ubuntu 22.04.3 LTS) and the HuggingFace transformers library, but does not provide a requirements.txt, Dockerfile, or list of library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes methodology in detail (hyperparameters, data processing, inference setup) but does not provide step-by-step reproduction instructions with commands. A GitHub repository is provided but the paper itself does not contain a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables II, III, and IV report only point estimates (e.g., '144 bugs fixed') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "McNemar tests are used for pairwise comparisons across all RQs, with exact p-values reported (e.g., 'p ≤ 1.27e−04', 'p ≈ 0.545', 'p ≤ 4.10e−08'). Full p-values are provided as supplemental material [41]."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Absolute effect sizes are reported throughout: e.g., 'plausibly repairs 49 (195 vs. 146) bugs more and correctly repairs 46 (144 vs. 98) more' (Section IV-B). Both absolute and relative differences are given with baseline context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes are dictated by the benchmarks (488 Defects4J, 162 HumanEval-Java, 90 GitBug-Java bugs) with no justification for whether these are adequate or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results are from single training runs. No standard deviation, variance across seeds, or spread measures are reported. While beam search inference is deterministic, training variance from different seeds is not explored."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: non-fine-tuned CodeLLaMA-7B (two prompt variants), full-parameter fine-tuning, deepseek-coder-6.7b, GPT-3.5, GPT-4, Jiang et al.'s fine-tuned Incoder-6B, and RAP-Gen (Tables II-IV)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GPT-3.5 and GPT-4 (Dec 2023 API versions), deepseek-coder-6.7b (2024), RAP-Gen (2023), and Jiang et al. (2023), all contemporary at time of experimentation."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 systematically ablates code representations (6 input/output pairs, Table II), isolating the effect of fault localization signals and buggy code preservation. RQ2 ablates the fine-tuning method (LoRA vs full-parameter, Table III)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four patch assessment metrics are used: plausible patches, exact match, AST match, and semantic match (Section III-C)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Semantic match assessment involves manual expert evaluation: 'the two first authors independently label all plausible but not AST/Exact match patches in a first round. For the patches the two first authors disagree upon, the third author breaks the tie' (Section III-C)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation benchmarks (Defects4J, HumanEval-Java, GitBug-Java) are all separate from the fine-tuning dataset (Megadiff). The authors verified no identical overlap exists and excluded 3 partially overlapping samples (Section V-D)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (3 benchmarks), per code representation (6 pairs in Table II), per fine-tuning method (Table III), and per model (Table IV). Multi-location bug results are also separately discussed."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses catastrophic forgetting on HumanEval-Java where fine-tuned IR3xOR2 performs worse than the baseline (Section IV-A). The paper also discusses why representations without fault localization fail to perform (rows 3-5 of Table II)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: IR1xOR1/OR3/OR4 representations 'perform considerably worse than both the baseline and other code representations' (Section IV-A). Fine-tuned IR3xOR2 underperforms baseline on HumanEval-Java. LoRA underperforms full fine-tuning for deepseek-coder on Defects4J and GitBug-Java."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 144 Defects4J, 109 HumanEval-Java, and 20 GitBug-Java bugs fixed, 'outperforming all baselines.' Tables II-IV confirm all numbers and the outperformance claims, with statistical significance tests supporting them."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'code representations with repair signals allow the LLM to synthesize patches more effectively') are supported by controlled ablation: 6 code representations evaluated with identical hyperparameters, isolating the representation variable. LoRA vs FFT comparisons also use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Program Repair' generally, but all experiments are limited to Java. While the threats to validity acknowledge 'the focus on a single programming language' and 'the focus on a single LLM,' the title and framing suggest broader applicability than demonstrated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses catastrophic forgetting as an alternative explanation for HumanEval-Java results (Section IV-A), overfitting as the reason full fine-tuning underperforms (Section IV-B), and fine-tuning dataset distribution mismatch between Megadiff and HumanEval-Java."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between plausible patches (pass all tests — the proxy) and correct patches (exact/AST/semantic match with developer patches — closer to ground truth), reporting both separately and discussing why test-passing alone is insufficient."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions: 'CodeLlama-7b', 'gpt-3.5-turbo-0613', 'gpt-4-0613', 'deepseek-coder-6.7b-base', 'gpt-3.5-turbo-0125' with training cutoff dates (Section III-B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows the exact prompt used for GPT-3.5 and GPT-4. Figures 2 and 3 show the exact input/output code representations used for fine-tuned models. All information sent to models is reproducible from these figures."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters: learning rate 5e-4 with cosine decay, max length 1024, epochs 2, batch size 16/GPU, Adam_W optimizer, LoRA rank 8, alpha 16, dropout 0.05, q_proj and v_proj layers, beam size 10. Full fine-tuning uses learning rate 2e-5 (Section III-B)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. RepairLLaMA is a straightforward fine-tuning and single-pass inference approach with no iterative repair, tool use, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The Megadiff processing pipeline is described: extract function pairs → eliminate non-single-function changes → remove duplicates via textual comparison → compute code representations → filter by total length < 1024 tokens → resulting in 30,000–50,000 pairs (Section III-B)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D 'Threats to Validity' provides substantive discussion of both internal threats (data leakage, fine-tuning overlap) and external threats (single language, single LLM)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: 3 Megadiff samples overlap with Defects4J (excluded); pre-training data may include test benchmarks (mitigated with GitBug-Java and HumanEval-Java); single language (Java); single base model (mitigated with deepseek-coder comparison); catastrophic forgetting on HumanEval-Java."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes to: Java bugs, functional bugs with failing test cases, single-function bugs, intra-procedural fixes (Section II-B). External threats state 'our results may not generalize to other languages' and 'the focus on a single LLM' (Section V-D)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Source code, models, and artifacts are published at the GitHub repository. All three benchmarks (Defects4J, HumanEval-Java, GitBug-Java) and the fine-tuning dataset (Megadiff) are publicly available. Statistical test results are provided as supplemental material [41]."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The Megadiff dataset origin and processing are described (Section III-B). The three benchmarks are cited with their collection methodology papers. The specific subsets used (e.g., 488 single-function Defects4J bugs from 835 total) are explained."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (Defects4J, HumanEval-Java, GitBug-Java) and an existing dataset (Megadiff)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from Megadiff to fine-tuning data is documented with filtering stages: extract function pairs → remove non-single-function changes → deduplicate → compute representations → length filter (< 1024 tokens) → 30K–50K pairs (Section III-B). Benchmark subset selection is also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VIII acknowledges supercomputing resources from Berzelius/National Supercomputer Centre at Linköping University, the Knut and Alice Wallenberg Foundation, and WASP (Wallenberg AI, Autonomous Systems and Software Program)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: KTH Royal Institute of Technology (Silva, Monperrus) and NC State University (Fang). These are academic institutions with no commercial stake in the evaluated models."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding from the Wallenberg Foundation and WASP are academic/governmental research funding bodies with no commercial interest in the performance of CodeLLaMA, GPT-3.5, or GPT-4."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Training data cutoffs are explicitly stated: 'CodeLlama-7b (September 2022), gpt-4-0613 (September 2021), gpt-3.5-turbo-0613 (September 2021)' (Section III-B)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive discussion: Megadiff vs Defects4J overlap was checked, 3 overlapping samples found and excluded (Section V-D). GitBug-Java contains only post-cutoff bugs. HumanEval-Java and GitBug-Java were constructed after Megadiff. Pre-training contamination is also discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that Defects4J bugs may be in LLM pre-training data and mitigates this by evaluating on GitBug-Java (2023 bugs, all after model training cutoffs) and HumanEval-Java. They also cite Ramos et al. [45] on LLM benchmark memorization (Section V-D)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The manual patch assessment by authors is not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study with automated and expert-assessed patch evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or time per bug is reported. The paper mentions inference uses 'a single A100 40GB GPU' but provides no wall-clock time or cost figures."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is stated (4xA100 40GB for training, single A100 for inference) but total GPU hours, training time, or API costs for GPT experiments are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single training run per configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. Results are presented without indicating whether they come from one or multiple training runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are stated (learning rate 5e-4, LoRA rank 8, etc.) but no description of how these were selected, whether any search was conducted, or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The best code representation (IR4xOR2) is selected based on systematic evaluation of all 6 representation pairs with results fully reported in Table II. Selection is transparent and based on evaluation benchmark performance."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "McNemar tests are performed for multiple pairwise comparisons across representations and models, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is applied or mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare RepairLLaMA against their own implementations of baselines and prompt designs for GPT models without acknowledging author-evaluation bias. Comparison with RAP-Gen uses the original authors' reported numbers, which is fairer."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 explicitly compares LoRA (4M parameters, 0.39% of full model) vs full-parameter fine-tuning (7B parameters), showing LoRA achieves better performance with ~1600x fewer parameters. The paper also notes RepairLLaMA generates 10 patches vs RAP-Gen's 100."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Defects4J, HumanEval-Java, or GitBug-Java actually measure real-world program repair capability. While it notes distribution differences between benchmarks (HumanEval-Java vs Megadiff), it does not question construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. RepairLLaMA is a direct fine-tuning approach with single-pass inference, no iterative agent or tool-based scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Training cutoffs are stated for all models, and GitBug-Java 'exclusively contains bugs from after the training data cutoff date of all models used in our experiments' (Section III-B). HumanEval-Java is also used to mitigate this concern."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses fault localization assumptions, proposing a more realistic region-based FL instead of perfect multi-line FL assumed by prior work. They argue this prevents information leakage from unrealistic FL assumptions (Section V-B)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors 'meticulously compare the samples in our fine-tuning dataset, Megadiff, with those in Defects4J' and found 3 overlapping samples which were excluded. HumanEval-Java and GitBug-Java were constructed after Megadiff (Section V-D)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Textual comparison of Megadiff against Defects4J was performed as a concrete detection method, identifying 3 samples where 'patch includes a function also found in Megadiff samples' (Section V-D). Temporal separation was used for GitBug-Java."
       }
     }
   },
   "claims": [
     {
       "claim": "RepairLLaMA with IR4xOR2 correctly fixes 144 Defects4J v2, 109 HumanEval-Java, and 20 GitBug-Java bugs (semantic match), outperforming all baselines.",
       "evidence": "Tables II, III, and IV report these numbers. Statistical significance tests confirm differences with p ≤ 1.27e−04 for most comparisons (Section IV-A, supplemental material [41]).",
       "supported": "strong"
     },
     {
       "claim": "Code representations with fault localization signals are essential for effective fine-tuning for program repair.",
       "evidence": "Table II rows 3-5 (no FL) perform considerably worse than rows 6-8 (with FL). IR1xOR1 fixes 45 Defects4J bugs vs IR4xOR2's 144. The difference is statistically significant (Section IV-A).",
       "supported": "strong"
     },
     {
       "claim": "Parameter-efficient fine-tuning (LoRA) outperforms full-parameter fine-tuning for CodeLLaMA-7B on program repair.",
       "evidence": "Table III: LoRA fixes 144 vs FFT's 98 Defects4J bugs (semantic match), 109 vs 100 HumanEval-Java, 20 vs 13 GitBug-Java. Statistical significance p ≤ 1.29e−03 (Section IV-B).",
       "supported": "strong"
     },
     {
       "claim": "The benefit of parameter-efficient fine-tuning varies across base models.",
       "evidence": "For deepseek-coder-6.7b, full fine-tuning outperforms LoRA on Defects4J (138 vs 128) and GitBug-Java (19 vs 13), while LoRA outperforms on HumanEval-Java (124 vs 119). Table III, Section IV-B.",
       "supported": "strong"
     },
     {
       "claim": "RepairLLaMA outperforms GPT-3.5 and GPT-4 on Defects4J and GitBug-Java.",
       "evidence": "Table IV: RepairLLaMA fixes 144 vs GPT-4's 72 (semantic match) on Defects4J, and 20 vs 10 on GitBug-Java. Statistical significance p ≤ 4.10e−08 (Section IV-C).",
       "supported": "strong"
     },
     {
       "claim": "RepairLLaMA's region-based fault localization enables repair of multi-location bugs.",
       "evidence": "51 multi-location bugs correctly repaired. Examples in Figures 5 (Math-86, two distant edit locations) and 6 (STRONGEST_EXTENSION, multiple modifications). Section IV-A.",
       "supported": "moderate"
     },
     {
       "claim": "LoRA acts as a regularizer preventing overfitting during fine-tuning.",
       "evidence": "The hypothesis is stated in Section IV-B citing Fu et al. [21], but no direct overfitting analysis (e.g., training vs validation loss curves) is provided. The claim is inferred from performance differences between LoRA and FFT.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance across training runs",
       "detail": "All results appear to be from single training runs per configuration. Without seed sensitivity analysis, it is impossible to assess whether the reported performance differences are stable or artifacts of a particular training run."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "McNemar tests are run for many pairwise comparisons across 6+ models without applying Bonferroni or similar corrections, inflating the risk of false positives in significance claims."
     },
     {
       "flag": "Missing compute budget",
       "detail": "Total GPU hours and training time per configuration are not reported, making it difficult to assess the true computational cost of the approach despite claims about efficiency."
     }
   ],
   "cited_papers": [
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "relevance": "Studies fine-tuning LLMs for program repair with code representations, a direct predecessor to RepairLLaMA's approach."
     },
     {
       "title": "RAP-Gen: Retrieval-augmented patch generation with CodeT5 for automatic program repair",
       "authors": ["W. Wang", "Y. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2023,
       "relevance": "State-of-the-art retrieval-augmented full-parameter fine-tuning baseline for program repair that RepairLLaMA outperforms."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu", "P. Wallis", "Z. Allen-Zhu", "Y. Li", "S. Wang", "L. Wang", "W. Chen"],
       "year": 2021,
       "relevance": "The parameter-efficient fine-tuning technique at the core of RepairLLaMA's approach."
     },
     {
       "title": "Code LLaMA: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "The base LLM used by RepairLLaMA, demonstrating code-specific pre-training with infilling support."
     },
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Demonstrates conversational/agent-based LLM program repair, contrasted with RepairLLaMA's fine-tuning approach."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["J. Yang", "C. E. Jimenez", "A. Wettig", "K. Lieret"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Agent-based approach to program repair using LLMs with tool access, representing the alternative paradigm to fine-tuning."
     },
     {
       "title": "An empirical study on fine-tuning large language models of code for automated program repair",
       "authors": ["K. Huang", "X. Meng", "J. Zhang", "Y. Liu"],
       "year": 2023,
       "relevance": "Studies code representations and evaluation metrics for fine-tuning LLMs for repair, but assumes unrealistic perfect multi-line fault localization."
     },
     {
       "title": "DeepSeek-Coder: When the large language model meets programming",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Alternative code LLM used as a second base model to evaluate RepairLLaMA's generalizability across architectures."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Explores zero-shot LLM-based program repair, showing the baseline capability without fine-tuning."
     },
     {
       "title": "On the effectiveness of parameter-efficient fine-tuning",
       "authors": ["Z. Fu", "H. Yang", "A. M.-C. So", "W. Lam"],
       "year": 2023,
       "relevance": "Theoretical foundation for why PEFT helps prevent overfitting, cited to support RepairLLaMA's regularization claims."
     },
     {
       "title": "Are large language models memorizing bug benchmarks?",
       "authors": ["D. Ramos", "C. Mamede", "K. Jain", "P. Canelas"],
       "year": 2024,
       "arxiv_id": "2411.13323",
       "relevance": "Directly addresses benchmark contamination risk in LLM program repair evaluation, a key validity concern."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Agent-based autonomous program repair approach representing the alternative paradigm to RepairLLaMA's fine-tuning."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released repair adapter can be used by practitioners for Java bug fixing with CodeLLaMA, though limited to single-function bugs."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Showing a 7B fine-tuned model outperforms GPT-4 is somewhat surprising but consistent with the broader fine-tuning literature."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; focused on improving automated program repair."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; straightforward empirical comparison of fine-tuning approaches."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code, models, and artifacts are published on GitHub, allowing replication and use of the repair adapter."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Monperrus lab at KTH is well-known in the program repair community but not a household name in broader AI discourse."
     }
   }
 }