ai-research-survey

scan.json (30874B)

---

 {
   "paper": {
     "title": "Accelerating Automatic Program Repair with Dual Retrieval-Augmented Fine-Tuning and Patch Generation on Large Language Models",
     "authors": [
       "Hanyang Guo",
       "Xiaoheng Xie",
       "Hong-Ning Dai",
       "Peng Di",
       "Yu Zhang",
       "Bishenghui Tao",
       "Zibin Zheng"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2507.10103"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Appendix H provides a link to https://anonymous.4open.science/r/SelRepair-5F1D/. However, this is an anonymous review platform link (anonymous.4open.science), which is ephemeral and used during peer review — not a permanent repository or archive. The schema requires 'a working URL or archive.' Anonymous review links are temporary and may be taken down after the review process. Under strict interpretation, this does not constitute a reliable, permanent code release. The paper is also an arXiv preprint, suggesting it is still under review."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two public datasets (Tufano dataset from Tufano et al. 2019, and VulRepair dataset from Fu et al. 2022) which are publicly available benchmarks. The enterprise dataset is not released, but the primary benchmarks are public."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 states 'All experiments are conducted on a server configured with 4 GPUs of NVIDIA GeForce RTX 3090' and specifies the base model (StarCoder2-7B), optimizer (Adam), learning rate (5e-5), context window sizes, and fine-tuning epochs. Sufficient environment detail is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is available via anonymous link and hyperparameters are reported, no step-by-step reproduction instructions (e.g., README commands to run experiments) are explicitly provided in the paper text itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported anywhere in the paper. All results in Tables 1-5 are point estimates only (e.g., '26.29% EM')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. The paper makes comparative claims (e.g., 'SelRepair achieves new SoTA performance... outperforming other SoTA LLMs') by comparing point estimates without any p-values, t-tests, or other significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with full baseline context throughout. For example, Table 1 shows SelRepair at 26.29% EM vs RAP-Gen at 24.80% EM, and the text reports percentage improvements like '6.01%' improvement over RAP-Gen. The schema explicitly states: 'A paper that says 12% improvement over baseline (from 45% to 57%) provides enough context for YES.' Both absolute scores and relative improvements are consistently provided across all comparisons in Tables 1-3, giving readers sufficient context to assess the magnitude of effects. Formal effect sizes like Cohen's d are not required by the schema."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The test sets (5,735 samples for Tufano Subset 1, 6,447 for Subset 2, 821 for VulRepair, 200 for enterprise dataset) are not explicitly justified as to why these sizes are sufficient for the claims made. The test set sizes come directly from public dataset splits without discussion of adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be single-run point estimates. There is no mention of multiple runs or seed variation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: GPT-3.5, GPT-4o, DeepSeek-R1-Distill, RAP-Gen, and several ablations of the proposed method (SelRepairLlama, SelRepairT5, SelRepairLoRA). Comparisons are shown in Tables 1 and 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent state-of-the-art systems: GPT-4o, DeepSeek-R1-Distill (2025), RAP-Gen (ESEC/FSE 2023). The baselines are contemporary and include recent general-purpose LLMs as well as specialized APR approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "A thorough ablation study is presented in Table 2 (Section 4.3, RQ2), testing five configurations: without RAG and fine-tuning, without fine-tuning, without RAG, without SR (semantic retriever), and without SSDR (structure/dependency retriever). All components are shown to contribute."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Exact Match (EM), BLEU-4, and CodeBLEU. These measure exact correctness, n-gram similarity, and code-specific quality respectively."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation is relevant to APR — automated metrics like EM only capture exact string matches and miss semantically correct alternative fixes. BLEU and CodeBLEU measure surface similarity, not semantic correctness. Human evaluation of patch quality would strengthen claims about repair effectiveness. The schema says applies=false only 'if human evaluation is clearly irrelevant to the claims.' For code repair, it is not clearly irrelevant — the paper itself acknowledges in the threats section that EM and BLEU may not fully reflect repair correctness. The paper could reasonably be expected to include human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly describes train/validation/test splits: 'we split 80% of the dataset as a training set, 10% as a validation set, and 10% as a test set' (Appendix E.1). Separate held-out test sets are used for final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset and code length (Tufano Subset 1 for <50 tokens, Tufano Subset 2 for 50-100 tokens, and VulRepair for C/C++). Table 3 further breaks down by RAG threshold settings, showing performance variation across configurations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section F.2 provides a case study discussing failure modes of other approaches (SelRepairT5 and SelRepairLoRA generate same code as buggy, SelRepairLlama misunderstands the method, GPT-3.5/4o make invalid modifications). Limitations note that the method fails on cross-method/cross-module bugs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2) explicitly shows that 'w/o RAG & Ft' and 'w/o Ft' achieve 0.00 EM, and SelRepairLlama performs poorly (5.96 EM on Subset 1). Table 3 shows that certain threshold settings hurt performance. The paper also notes performance close to (but not exceeding) RAP-Gen on Defects4J V2.0."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '26.29% and 17.64% in terms of exact match (EM) on different datasets while reducing inference time by at least 6.42%' are directly supported by Table 1 (EM scores) and Table 3 (inference time reductions of 6.42%, 13.77%, and 9.95%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ablation studies (Table 2) to make causal claims about which components contribute to performance. Each module (SR, SSDR, fine-tuning) is systematically removed and the resulting performance drop is measured, constituting controlled single-variable manipulation adequate for component-level causal inference."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract use broad language ('Accelerating Automatic Program Repair') but the evaluation is limited to Java and C/C++ datasets at the method level. The paper acknowledges in the threats to validity that 'SelRepair showed promise in repairing Java programs, its effectiveness with other languages like Python or JavaScript is untested,' but the abstract does not adequately bound this scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Appendix G) discusses generic limitations (external validity about language coverage, construct validity about metrics) but does not seriously consider alternative explanations for why SelRepair outperforms baselines. For example, the dataset used to fine-tune SelRepair may overlap more with the test set than RAP-Gen's training data, which is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The base model 'StarCoder2-7B' is specified. However, 'GPT-3.5' and 'GPT-4o' are used without version numbers, snapshot dates, or API version identifiers, which significantly affects reproducibility given that model behavior changes across versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.2 shows the code-only prompt format as a template: '[BUG] RBC1 [FIX] RFC1 ... [BUG] BC [FIX]' where RBC1, RFC1, BC are placeholders for actual code. Figure 5 shows the GPT-3.5/GPT-4o prompt template with placeholder sections ('Target Buggy Code', 'Retrieved Bug-fix Pairs'). The schema explicitly states: 'A prompt TEMPLATE with placeholders (e.g., [Task Description]) does NOT count unless the actual fill values are also provided — the reader must be able to reconstruct every prompt sent to the model.' While Figure 6 shows one concrete example, the reader cannot reconstruct the full set of prompts used across thousands of test samples. The prompt structure is shown but not the actual prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 reports: fine-tuning epochs (3 for large LLM, 50 for CodeT5), context window sizes (512, 1024, 1500 tokens for different datasets), learning rate (5e-5), optimizer (Adam), RAG thresholds (0.9 for Subset 1, 0.8 for Subset 2 and VulRepair), and hardware (4x NVIDIA RTX 3090)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The dual RAG architecture is described in detail in Section 3.1 including the hybrid retriever algorithm (Algorithm 1 in Appendix C), AST traversal procedure (Algorithm 2 in Appendix D), the RAG selection gate mechanism with threshold, and how retrieved pairs are incorporated into the fine-tuning prompt."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.1 describes dataset construction: random sampling of 1,000 samples for RAG codebase, 80/10/10 train/validation/test split for the remaining samples, and filtering of 'invalid samples, such as samples that were null' from VulRepair. Dataset sizes are reported in Table 4."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The paper has two limitations sections: a 'Limitations' section at the end of the main paper (after the conclusion) and a dedicated 'Threats to Validity' appendix (Appendix G) covering internal, external, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity section names specific threats: (1) errors in bug-fix dataset and overfitting risk during fine-tuning; (2) coarse-grained threshold settings; (3) untested generalization to languages other than Java and C/C++; (4) reliance on EM and BLEU-4 metrics that may not fully reflect repair correctness. These are moderately specific to this paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section states the method is 'limited by datasets focused on individual methods' and 'effectiveness with other languages like Python or JavaScript is untested.' The paper explicitly identifies what was not tested (cross-method/cross-module bugs, multiple programming languages beyond Java and C/C++)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the Tufano and VulRepair benchmarks are publicly available as original datasets, the paper's specific processed versions (exact random splits, RAG codebase selection of 1,000/2,000 samples) are not verifiably released. The anonymous repository link (anonymous.4open.science) is ephemeral. The enterprise dataset of 200 samples is explicitly not released ('we intend to open-source this benchmark in the future'). For independent verification of the reported results, one needs the exact data splits and enterprise data used — the original public benchmarks alone are insufficient since random sampling was used to create splits."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.1 describes how each dataset was constructed: the Tufano dataset 'is collected from fix commit records from GitHub' with known splits by code length; VulRepair 'consists of bug-fix pairs combined by CVE-Fixes and BigVul.' The enterprise dataset 'consists of 200 semantic bug-fix pairs caused by enterprise developers in real development scenarios.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "There are no human participants in this study. The data source is a standard public benchmark (Tufano dataset from GitHub commit history) and an enterprise software dataset. NA for this criterion."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: random sampling of RAG codebase, 80/10/10 splits, filtering of null/invalid samples from VulRepair. Table 4 reports the final dataset sizes for each split, allowing verification that the pipeline is consistent."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments or funding disclosure section is present in the paper. It is unclear whether this work was funded by Ant Group (which employs two authors) or by academic grants. The absence of a funding statement is treated as NO."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed on the title page: Xiaoheng Xie, Peng Di, and Yu Zhang are from Ant Group (a major tech company). This affiliation is disclosed, even though the paper does not explicitly discuss any potential conflict of interest this may represent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "With three authors from Ant Group and the evaluation including an enterprise dataset 'from a software enterprise,' there is a plausible conflict of interest. The enterprise dataset may be from Ant Group itself. However, since funding is not disclosed at all, independence cannot be verified, and NO is the appropriate answer."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or disclosure of financial interests (patents, equity) is present in the paper. The absence of such a statement means this criterion is not satisfied."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for GPT-3.5 and GPT-4o is not stated. For StarCoder2-7B (the base model for SelRepair), the training cutoff is not mentioned in the paper. The Tufano dataset was published in 2019 and is widely used, making contamination risk real for both GPT models and StarCoder2."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential train-test overlap for GPT-3.5, GPT-4o, or StarCoder2-7B with the Tufano benchmark dataset. Section F.3 does note that RAP-Gen's training data 'constructed by the same projects as Defects4J... may cause data leaks,' but this concern is not applied to SelRepair or the GPT baselines."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The Tufano dataset (from 2019) is a widely used benchmark that was almost certainly included in the training data of GPT-3.5, GPT-4o, and StarCoder2-7B (all trained on large code corpora). The paper does not address this contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. All evaluation uses automated metrics on code datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. IRB approval is not applicable."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Demographics are not applicable."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Inclusion/exclusion criteria for participants are not applicable."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Randomization of participants is not applicable."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Blinding is not applicable."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. Attrition is not applicable."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports inference time reductions (e.g., '6.42% less than no threshold setting') and average input token lengths for different configurations. This constitutes a practical cost/efficiency analysis, though monetary API costs are not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the hardware is stated (4x NVIDIA RTX 3090), total GPU hours, training time, or total compute budget are not reported. The paper does not quantify how much compute was required for the fine-tuning process."
       }
     }
   },
   "claims": [
     {
       "claim": "SelRepair achieves state-of-the-art performance of 26.29% EM on Tufano Subset 1 and 17.64% EM on Tufano Subset 2, outperforming all compared approaches.",
       "evidence": "Table 1 in Section 4.2 shows SelRepair achieving 26.29 EM and 17.64 EM on the two Tufano subsets, compared to RAP-Gen (24.80, 15.84), GPT-3.5 (2.58, 1.72), and GPT-4o (0.17, 0.00).",
       "supported": "strong"
     },
     {
       "claim": "The RAG selection gate reduces inference time by at least 6.42% while maintaining or improving APR performance.",
       "evidence": "Table 3 (Section 4.4) shows that with threshold 0.9, inference time decreases by 6.42%, 29.68%, and 15.96% compared to no threshold in the three datasets, while the EM score equals or exceeds the no-threshold baseline.",
       "supported": "strong"
     },
     {
       "claim": "Both semantic retrieval (SR) and structural/dependency retrieval (SSDR) contribute to APR performance.",
       "evidence": "Ablation study in Table 2 (Section 4.3) shows SelRepair outperforms 'w/o SR' by 2.82% and 'w/o SSDR' by 18.00% on Tufano Subset 1 in terms of EM. All ablated versions perform worse than the full SelRepair.",
       "supported": "strong"
     },
     {
       "claim": "Full-parameter fine-tuning contributes more to APR performance than PEFT (LoRA) fine-tuning.",
       "evidence": "Table 1 shows SelRepairLoRA achieves 22.62 EM vs. SelRepair's 26.29 EM on Tufano Subset 1. Table 2 shows that 'w/o Fine-tuning' achieves 0.00 EM, confirming fine-tuning is essential.",
       "supported": "moderate"
     },
     {
       "claim": "SelRepair generalizes to real-world enterprise scenarios, generating 59 correct patches from 200 enterprise bug-fix pairs.",
       "evidence": "Figure 2 (Section 4.5) shows SelRepair generates 59 correct patches on the enterprise dataset, compared to RAP-Gen's 1 correct patch. However, the enterprise dataset is not public and evaluation details are limited.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "SelRepair, a novel automated program repair framework combining full-parameter fine-tuning of StarCoder2-7B with a dual RAG module (semantic and syntactic/structural retrievers), achieves state-of-the-art performance on Java and C/C++ APR benchmarks with 26.29% and 17.64% exact match on Tufano subsets. The RAG selection gate mechanism controls prompt length and reduces inference time by 6.42-29.68% while maintaining or improving repair accuracy. Ablation studies confirm that both retrieval components and fine-tuning are necessary, with the system substantially outperforming both prompt-engineering baselines (GPT-3.5, GPT-4o) and smaller fine-tuned models (RAP-Gen). An enterprise evaluation with 200 real-world bug-fix pairs shows 59 correct patches, demonstrating practical applicability.",
   "red_flags": [
     {
       "flag": "No statistical uncertainty quantification",
       "detail": "All results in Tables 1-5 are single-run point estimates with no confidence intervals, error bars, or variance reported across multiple seeds. Claims of SOTA superiority (e.g., SelRepair 26.29% vs. RAP-Gen 24.80% on Tufano Subset 1 — a 1.49 percentage point difference) are made without testing whether the differences are statistically significant."
     },
     {
       "flag": "Benchmark contamination not addressed",
       "detail": "The Tufano dataset (published 2019) is widely used in code LLM training corpora. Both StarCoder2-7B (the base model for SelRepair) and GPT-3.5/GPT-4o were trained on large code datasets that almost certainly include this benchmark. The paper does not discuss contamination risk, and the paper acknowledges RAP-Gen has data leakage concerns on Defects4J but does not apply the same scrutiny to SelRepair."
     },
     {
       "flag": "GPT baseline versions unspecified",
       "detail": "GPT-3.5 and GPT-4o are used as baselines without specifying API version or snapshot date. These models have different versions with varying capabilities, and results may not be reproducible as the referenced model version cannot be determined."
     },
     {
       "flag": "Undisclosed funding and potential conflict of interest",
       "detail": "Three of the seven authors are affiliated with Ant Group. The enterprise dataset used in RQ4 may be from Ant Group itself, creating a potential evaluation bias. No funding disclosure or competing interests statement is present in the paper."
     },
     {
       "flag": "Enterprise dataset not publicly available",
       "detail": "The enterprise dataset of 200 bug-fix pairs used in RQ4 is not publicly released, making those results unverifiable. The paper states 'we intend to open-source this benchmark in the future' but this is a promise, not a release."
     },
     {
       "flag": "Overly broad title relative to scope",
       "detail": "The paper is titled 'Accelerating Automatic Program Repair' generally, but evaluation is restricted to Java and C/C++ at the method level only. The limitations section acknowledges this but the abstract does not sufficiently bound the generalizability claim."
     }
   ],
   "cited_papers": [
     {
       "title": "RAP-Gen: Retrieval-augmented patch generation with CodeT5 for automatic program repair",
       "authors": [
         "Weishi Wang",
         "Yue Wang",
         "Shafiq Joty",
         "Steven C.H. Hoi"
       ],
       "year": 2023,
       "relevance": "The primary competing baseline in this paper; a RAG-based APR approach using CodeT5 and semantic similarity retrieval that SelRepair improves upon."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": [
         "Baptiste Rozière",
         "Jonas Gehring",
         "Fabian Gloeckle"
       ],
       "year": 2024,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as the base model for one SelRepair variant (SelRepairLlama), relevant to code generation and LLM capability evaluation."
     },
     {
       "title": "StarCoder 2 and the Stack v2: The next generation",
       "authors": [
         "Anton Lozhkov",
         "Raymond Li",
         "Loubna Ben Allal"
       ],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "StarCoder2-7B is the base model used for SelRepair's full-parameter fine-tuning; relevant to code LLM evaluation in the survey."
     },
     {
       "title": "UniXcoder: Unified cross-modal pre-training for code representation",
       "authors": [
         "Daya Guo",
         "Shuai Lu",
         "Nan Duan",
         "Yanlin Wang",
         "Ming Zhou",
         "Jian Yin"
       ],
       "year": 2022,
       "relevance": "The code embedding model used in SelRepair's retrieval module; directly relevant to LLM-based code tooling."
     },
     {
       "title": "An empirical study on fine-tuning large language models of code for automated program repair",
       "authors": [
         "Kai Huang",
         "Xiangxin Meng",
         "Jian Zhang",
         "Yang Liu",
         "Wenjie Wang",
         "Shuhao Li",
         "Yuqing Zhang"
       ],
       "year": 2023,
       "relevance": "Empirical study on fine-tuning LLMs for APR that informs the methodology of this paper; directly relevant to the survey's focus on LLM evaluation in software engineering."
     },
     {
       "title": "VulRepair: a T5-based automated software vulnerability repair",
       "authors": [
         "Michael Fu",
         "Chakkrit Tantithamthavorn",
         "Trung Le",
         "Van Nguyen",
         "Dinh Phung"
       ],
       "year": 2022,
       "relevance": "Provides the C/C++ vulnerability repair dataset used in this paper's evaluation; relevant as a benchmark paper for LLM-based program repair."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": [
         "Chunqiu Steven Xia",
         "Yinlin Deng",
         "Soren Dunn",
         "Lingming Zhang"
       ],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "A prominent LLM-based software engineering agent paper that provides context for the agentic approach to program repair."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": [
         "Nan Jiang",
         "Kevin Liu",
         "Thibaud Lutellier",
         "Lin Tan"
       ],
       "year": 2023,
       "relevance": "Empirical evaluation of code LLMs for automated program repair, directly relevant to the survey's focus on LLM-based SE tool evaluation."
     },
     {
       "title": "RepairLlama: Efficient representations and fine-tuned adapters for program repair",
       "authors": [
         "André Silva",
         "Sen Fang",
         "Martin Monperrus"
       ],
       "year": 2024,
       "arxiv_id": "2312.15698",
       "relevance": "PEFT-based approach for LLM program repair that this paper compares against and distinguishes from via full-parameter fine-tuning."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": [
         "René Just",
         "Darioush Jalali",
         "Michael D. Ernst"
       ],
       "year": 2014,
       "relevance": "Standard APR benchmark used for supplementary evaluation (Appendix F.3); widely used in the SE community for evaluating program repair tools."
     },
     {
       "title": "Retrieval-based prompt selection for code-related few-shot learning",
       "authors": [
         "Noor Nashid",
         "Mifta Sintaha",
         "Ali Mesbah"
       ],
       "year": 2023,
       "relevance": "Prior RAG-based approach for code tasks that this paper builds upon and improves for APR with dual retrieval."
     },
     {
       "title": "Towards an understanding of large language models in software engineering tasks",
       "authors": [
         "Zibin Zheng",
         "Kaiwen Ning",
         "Jiachi Chen",
         "Yanlin Wang",
         "Wenqing Chen",
         "Lianghong Guo",
         "Weicheng Wang"
       ],
       "year": 2023,
       "arxiv_id": "2308.11396",
       "relevance": "Survey of LLMs applied to software engineering tasks; provides broader context for evaluating LLM capability in code-related tasks."
     }
   ]
 }