ai-research-survey

scan.json (29840B)

---

 {
   "paper": {
     "title": "Adapting Knowledge Prompt Tuning for Enhanced Automated Program Repair",
     "authors": [
       "Xuemeng Cai",
       "Lingxiao Jiang"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2504.01523"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in the Conclusion: 'Our source code and experimental data are publicly available at: https://github.com/Cxm211/k-prompt'. A working GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses six publicly available APR benchmark datasets (BugsInPy, Code Refinement, Defects4J, ManySStuBs4J-SStuBs, TFix, xCodeEval-APR-C), all of which are standard public benchmarks. The GitHub repository also claims to include experimental data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using OpenPrompt and PyTorch as frameworks and specifies hardware (NVIDIA A40 48GB GPU and H100 80GB GPU), but does not provide a requirements.txt, Dockerfile, or library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While source code is available on GitHub, the paper itself contains no step-by-step reproduction instructions. The paper describes the experimental design but does not provide commands or a README walkthrough in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results throughout the paper (Tables VI, VII, VIII, IX, X) are reported as single point estimates with no confidence intervals or error bars. The paper averages results over three runs but does not report standard deviations or any spread measure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are applied. The paper compares prompt tuning vs. fine-tuning by reporting raw numbers and percentages without any p-values, t-tests, or other statistical tests to verify that observed differences are not due to chance."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw percentage improvements (e.g., '87.33% on average') but these are computed as relative improvements rather than standardized effect sizes (Cohen's d, etc.). No proper effect size measures are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper selects 1% samples from large datasets to simulate data scarcity but does not justify why these particular sample sizes (260-1082 training instances) are sufficient for drawing conclusions. No power analysis is presented."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs experiments three times with different seeds and takes averages, but never reports standard deviations or any spread measure for these runs. Tables only show mean/average values without any variance information."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Fine-tuning is used as the baseline comparison throughout. Additionally, a 'Naive Copy' baseline (copying buggy code as the fix) is included. Results are compared across all six datasets."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is fine-tuning of the same pre-trained models (CodeT5+ 220M, CodeT5+ 770M, GPT-Neo 1.3B), which is the current standard approach for APR. Fine-tuning is the appropriate contemporary baseline for prompt tuning."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 and RQ3 serve as ablation studies. RQ2 compares hard vs. soft prompts and different prompt templates (BP1-BP7). RQ3 ablates individual types of domain knowledge (repair action, repair pattern, bug type, AST, error message, tags) and their combinations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Exact Match (EM), Syntactically Correct Patch (SC), and CodeBLEU. Multiple metrics address different aspects of repair quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Human evaluation IS relevant for APR. Exact Match only catches patches identical to a reference fix; there can be multiple valid fixes. Human evaluation of generated patches would strengthen the claims about repair effectiveness. The paper itself acknowledges EM may not capture the full fix ratio (Construct Validity section). The criterion is applicable — the paper could reasonably have included human evaluation of patches — but chose not to. The schema says applies=false only when 'human evaluation is clearly irrelevant to the claims.' For APR, where correctness of patches matters and automated metrics are known to be incomplete, human evaluation is relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly splits data into training, validation, and test sets in an 8:1:1 ratio (Table IV). Results in RQ1-RQ3 are reported on held-out test sets. RQ4 uses a fixed 500-instance test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all six individual datasets (BugsInPy, Code Refinement, Defects4J, ManySStuBs4J-SStuBs, TFix, xCodeEval), across four programming languages (Python, Java, JavaScript, C), and across three models separately."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where prompt tuning does not help (e.g., Code Refinement dataset where variable anonymization limits performance, xCodeEval where noisy domain knowledge reduces effectiveness). Section V.C discusses failure modes of combining multiple knowledge types."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: combining multiple domain knowledge types often does not lead to incremental gains (RQ3/Table IX); certain prompt templates (BP1, BP5, BP6, BP7) consistently underperform for GPT-Neo; Code Refinement dataset shows lower improvement due to variable anonymization."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claim of '87.33% average improvement over fine-tuning in data scarcity scenarios' is supported by Table VI results across the six datasets and three models. The claim about comprehensive evaluation across three LLMs, six datasets, and four languages is confirmed by the experimental design."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims (prompt tuning 'improves' performance over fine-tuning). The ablation design is adequate for this — they control for model architecture, datasets, and evaluation protocol, varying only the tuning method. The controlled single-variable manipulation supports the causal interpretation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges scope limitations: 'Our experimental results may be applicable to specific datasets and models only' (Section VI.B, External Validity). The conclusion also acknowledges evaluation is limited to 'four datasets, three relatively small pre-trained code models, and limited prompt templates.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The Threats to Validity section (Section VI.B) discusses generic threats: randomness in training (Internal Validity), EM metric limitations (Construct Validity), and limited generalizability (External Validity). However, these are methodological limitations, not alternative explanations for the observed results. The paper does not discuss specific alternative explanations such as: whether prompt tuning's advantage is simply due to having more trainable parameters per effective sample, whether the improvement is driven by the prompt providing a stronger inductive bias rather than domain knowledge per se, or whether the 87.33% improvement figure is inflated by the choice of data scarcity threshold. The Code Refinement variable anonymization discussion is a dataset-specific observation, not an alternative explanation for the main findings. Per the schema: 'A threats-to-validity section counts only if it discusses specific alternative explanations for the observed results, not just generic methodological limitations.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names and parameter counts (CodeT5+ 220M, CodeT5+ 770M, GPT-Neo 1.3B) and states they are loaded from 'official versions available on Hugging Face.' However, it does not specify exact Hugging Face model identifiers (e.g., Salesforce/codet5p-220m) or commit hashes. Per the schema: 'Marketing names like Gemini-2.5 or GPT-4o without a snapshot date or API version do NOT count as specified versions.' CodeT5+ 220M is a model family name with size, not a precise version — model weights on Hugging Face can be updated without changing the model name. These are open-weight models with specific sizes, which makes them more identifiable than API models, but the schema explicitly requires version-level specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Tables I and III show prompt TEMPLATES with placeholders ([X], [mask], [bugType], [repairAction], [repairPattern], [SOFT], etc.). Per the schema: 'A prompt TEMPLATE with placeholders (e.g., [Task Description]) does NOT count unless the actual fill values are also provided — the reader must be able to reconstruct every prompt sent to the model.' While [X] (buggy code) and [mask] (output slot) are structural model placeholders, the knowledge prompt templates use domain knowledge placeholders like [bugType], [repairAction], [repairPattern], [AST], [errorMessage], [tags] whose actual fill values are dataset-derived and not fully enumerated. The paper shows one example (Fig. 4) but does not provide the actual domain knowledge values used across all experimental instances. A reader cannot reconstruct every prompt sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table V provides a complete hyperparameter table including optimizer (AdamW), Adam epsilon (1e-8), initial learning rate (5e-5), LR scheduler (Linear), training epochs (10), temperature (1.0), top-p (0.9), beam number (5), and repetition penalty (1.0)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper performs standard prompt tuning and fine-tuning with LLMs; there is no agent, tool-use framework, or complex scaffolding involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C describes preprocessing: single-hunk fixes are treated as separate instances, data is split 8:1:1 into train/val/test, and for large datasets 1% is sampled using three distinct seeds with averages taken. The criteria for the 1% sampling and the single-hunk restriction are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI.B 'Threats to Validity' contains three subsections: Construct Validity, Internal Validity, and External Validity, each discussing specific methodological limitations with substantive content."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: Construct Validity discusses how EM may not capture all valid fixes and that prompt templates may not be optimal; Internal Validity discusses the impact of randomness in training and sampling; External Validity acknowledges that results may only apply to the tested datasets and models."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The conclusion explicitly states what the results do not show: evaluation is limited to 'four datasets, three relatively small pre-trained code models, and limited prompt templates'. External Validity also notes results 'may be applicable to specific datasets and models only' and that 'we may need more evaluation in the future to conclude that our results could be generalizable to all datasets and models.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets (BugsInPy, Code Refinement, Defects4J, ManySStuBs4J, TFix, xCodeEval) and states experimental data is available at the GitHub repository. Original benchmark data can be independently verified from the cited sources."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.C describes how datasets were selected (six commonly used APR benchmarks from literature), how they were processed (single-hunk fixes extracted, 8:1:1 split, 1% sampling for large datasets), and dataset statistics are provided in Table IV."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "There are no human participants. The study uses pre-existing benchmark datasets from real-world software projects. This is a purely computational study with no participant recruitment."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: starting from six benchmark datasets, applying the single-hunk restriction, splitting 8:1:1, and sampling 1% from large datasets using three seeds. Table IV shows final dataset statistics. The steps are clear and traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section states: 'This research is supported by the Ministry of Education, Singapore under its Academic Research Fund Tier 3 (Award ID: MOET32020-0004).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Both authors are affiliated with Singapore Management University (SMU), which is clearly stated on the title page. No commercial affiliations are present; the authors are not evaluating a product they have financial stake in."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the Ministry of Education, Singapore, a government academic funding body with no financial interest in prompt tuning vs. fine-tuning for APR. The disclaimer in Acknowledgments states the views expressed do not reflect those of the Ministry."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "The paper contains no competing interests statement or declaration of financial interests. While no obvious conflict exists (academic researchers at a university), the absence of an explicit competing interests declaration means this criterion is not met."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for the models used (CodeT5+, GPT-Neo). Since these models were trained on web-scraped code data, the cutoff dates are relevant for assessing whether the benchmark tasks could have appeared in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential overlap between the pre-training data of CodeT5+ or GPT-Neo and the test sets of the APR benchmarks used (Defects4J, etc.). The benchmarks are publicly available and could have been in pre-training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Some benchmarks like Defects4J and Code Refinement were published before the training cutoffs of CodeT5+ and GPT-Neo, making contamination plausible. The paper does not discuss whether the test instances could have appeared in pre-training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study evaluating LLMs on APR benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a computational study with no human subjects."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or cost-per-example are reported. The paper mentions hardware used (A40 and H100 GPUs) but does not report inference latency or computational cost of running the approach."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies the hardware used (NVIDIA A40 48GB GPU for CodeT5+ and H100 80GB GPU for GPT-Neo) but does not report total GPU hours, training time, or total compute budget. Given the comprehensive experiments across many configurations, the compute expenditure is non-trivial but unquantified."
       }
     }
   },
   "claims": [
     {
       "claim": "Prompt tuning achieves an average improvement of 87.33% over fine-tuning in data scarcity scenarios for APR tasks.",
       "evidence": "Table VI reports prompt tuning vs. fine-tuning EM rates across six datasets and three models. The 87.33% is stated as the average improvement for CodeT5+ models (Section V.A). GPT-Neo improves from 0% to 1.03%-54.17% EM rate.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt tuning consistently outperforms fine-tuning across all three base models (CodeT5+ 220M, CodeT5+ 770M, GPT-Neo 1.3B) and all six APR datasets.",
       "evidence": "Table VI shows prompt tuning outperforming fine-tuning on every dataset/model combination. Finding 1 and Finding 2 summarize these results.",
       "supported": "strong"
     },
     {
       "claim": "Soft basic prompts (SBP) generally outperform hard basic prompts (HBP) in prompt tuning for APR.",
       "evidence": "Table VII compares HBP, SBPinitialized, and SBPrandom across multiple datasets and models. SBP variants generally outperform HBP. Finding 3 summarizes this result.",
       "supported": "moderate"
     },
     {
       "claim": "Incorporating bug- and code-related domain knowledge in knowledge prompts generally improves prompt tuning performance.",
       "evidence": "Table IX compares basic prompts with various knowledge prompts for Defects4J, ManySStuBs4J, TFix, and xCodeEval. Most knowledge prompts outperform basic prompts. Finding 5 summarizes this.",
       "supported": "moderate"
     },
     {
       "claim": "Combining multiple types of domain knowledge does not lead to incremental gains and may reduce performance compared to single knowledge type.",
       "evidence": "Table IX shows that 'Repair Action + Repair Pattern' for Defects4J and 'Bug Type + AST' for ManySStuBs4J do not outperform single-knowledge prompts. The discussion in Section V.C attributes this to conflicting or noisy information.",
       "supported": "moderate"
     },
     {
       "claim": "Prompt tuning is particularly advantageous in extreme data scarcity (1-32 shot) scenarios.",
       "evidence": "Table X compares prompt tuning and fine-tuning with 1, 8, 16, and 32 training instances. In the 1-shot scenario, fine-tuned models achieve 0% EM while prompt-tuned models achieve non-zero repair rates. Fig. 5 shows trends across training sizes.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Prompt tuning consistently outperforms fine-tuning for Automated Program Repair (APR) in data scarcity scenarios, achieving an average 87.33% improvement over fine-tuning across CodeT5+ 220M and 770M models on six APR benchmarks. The approach, called knowledge prompt tuning, integrates bug- and code-related domain knowledge (repair actions, ASTs, bug types, error messages) into prompt templates, which generally enhances performance beyond basic prompt tuning. Soft prompts (with learnable tokens) outperform hard prompts (fixed natural language tokens) in APR generation tasks. The advantage of prompt tuning is especially pronounced in extreme data scarcity settings (1-32 training instances), where fine-tuned models often achieve 0% Exact Match while prompt-tuned models successfully repair some bugs.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper makes comparative claims ('outperforms', 'achieves improvement') based solely on point estimates without any statistical tests. Given the small test set sizes (32-135 instances per dataset) and multiple comparisons across 18+ model-dataset-prompt combinations, the lack of significance testing makes it impossible to distinguish real improvements from chance variation."
     },
     {
       "flag": "No variance reported for multi-run experiments",
       "detail": "The paper explicitly runs experiments three times with different seeds and takes averages, but never reports standard deviations or any spread measure. This makes it impossible to assess the stability of results or whether differences between methods are meaningful."
     },
     {
       "flag": "No benchmark contamination discussion",
       "detail": "Several benchmarks (Defects4J, Code Refinement) were published before the training cutoffs of CodeT5+ and GPT-Neo. If benchmark instances appeared in pre-training data, this would inflate performance measures and bias comparisons. The paper does not address this threat."
     },
     {
       "flag": "Small test sets limit reliability",
       "detail": "Some test sets are very small (ManySStuBs4J: 32 instances, Code Refinement: 65 instances, Defects4J: 64 instances). A 1% EM improvement on 32 instances represents less than one additional correctly repaired bug. Claims of consistent outperformance are fragile at these sample sizes without confidence intervals."
     },
     {
       "flag": "Compute cost not reported",
       "detail": "The paper proposes prompt tuning as a practical alternative to fine-tuning, but does not report training time, GPU hours, or total compute budget despite running hundreds of experiments across multiple models, datasets, and configurations on high-end GPUs (A40 and H100)."
     }
   ],
   "cited_papers": [
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": [
         "Y. Wang",
         "W. Wang",
         "S. Joty",
         "S. C. Hoi"
       ],
       "year": 2021,
       "relevance": "Core LLM for code used in the APR experiments; directly relevant to LLM-based code generation and program repair."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": [
         "C. S. Xia",
         "Y. Wei",
         "L. Zhang"
       ],
       "year": 2023,
       "relevance": "Evaluates LLMs for APR across benchmarks, directly relevant to the survey scope on LLM program repair methodology."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": [
         "N. Jiang",
         "K. Liu",
         "T. Lutellier",
         "L. Tan"
       ],
       "year": 2023,
       "relevance": "Empirical study fine-tuning LLMs for APR tasks; directly related to the methodology being compared in this paper."
     },
     {
       "title": "An empirical study on fine-tuning large language models of code for automated program repair",
       "authors": [
         "K. Huang",
         "X. Meng",
         "J. Zhang",
         "Y. Liu",
         "W. Wang",
         "S. Li",
         "Y. Zhang"
       ],
       "year": 2023,
       "relevance": "Comprehensive fine-tuning study for LLMs in APR; baseline methodology this paper compares against."
     },
     {
       "title": "No more fine-tuning? An experimental evaluation of prompt tuning in code intelligence",
       "authors": [
         "C. Wang",
         "Y. Yang",
         "C. Gao",
         "Y. Peng",
         "H. Zhang",
         "M. R. Lyu"
       ],
       "year": 2022,
       "relevance": "Directly evaluates prompt tuning for code intelligence tasks; foundational comparison work for the paper's approach."
     },
     {
       "title": "RAP-Gen: Retrieval-augmented patch generation with CodeT5 for automatic program repair",
       "authors": [
         "W. Wang",
         "Y. Wang",
         "S. Joty",
         "S. C. Hoi"
       ],
       "year": 2023,
       "relevance": "Uses retrieval-augmented prompts for APR with CodeT5; directly related to knowledge-enhanced approaches for program repair."
     },
     {
       "title": "The power of scale for parameter-efficient prompt tuning",
       "authors": [
         "B. Lester",
         "R. Al-Rfou",
         "N. Constant"
       ],
       "year": 2021,
       "relevance": "Foundational work on prompt tuning methodology that this paper extends to APR tasks."
     },
     {
       "title": "Assessing the effectiveness of vulnerability detection via prompt tuning: An empirical study",
       "authors": [
         "G. Lu",
         "X. Ju",
         "X. Chen",
         "S. Yang",
         "L. Chen",
         "H. Shen"
       ],
       "year": 2023,
       "relevance": "Evaluates prompt tuning for security vulnerability detection in code; relevant to LLM-based program analysis."
     },
     {
       "title": "VulRepair: A T5-based automated software vulnerability repair",
       "authors": [
         "M. Fu",
         "C. Tantithamthavorn",
         "T. Le",
         "V. Nguyen",
         "D. Phung"
       ],
       "year": 2022,
       "relevance": "T5-based fine-tuning for vulnerability repair; directly related to LLM-based APR methodology."
     },
     {
       "title": "Prefix-tuning: Optimizing continuous prompts for generation",
       "authors": [
         "X. L. Li",
         "P. Liang"
       ],
       "year": 2021,
       "relevance": "Foundational work on soft/continuous prompt tuning methods that underpin the approach evaluated in this paper."
     },
     {
       "title": "TBar: Revisiting template-based automated program repair",
       "authors": [
         "K. Liu",
         "A. Koyuncu",
         "D. Kim",
         "T. F. Bissyande"
       ],
       "year": 2019,
       "relevance": "Template-based APR baseline system that LLM-based approaches are compared against."
     },
     {
       "title": "How effective are neural networks for fixing security vulnerabilities",
       "authors": [
         "Y. Wu",
         "N. Jiang",
         "H. V. Pham",
         "T. Lutellier",
         "J. Davis",
         "L. Tan"
       ],
       "year": 2023,
       "relevance": "Evaluates neural APR for security vulnerability repair; relevant to LLM-based program repair methodology."
     }
   ]
 }