ai-research-survey

scan.json (34135B)

---

 {
   "paper": {
     "title": "Synthetic Code Surgery: Repairing Bugs and Vulnerabilities with LLMs and Synthetic Data",
     "authors": [
       "David de-Fitero-Dominguez",
       "Antonio Garcia-Cabot",
       "Eva Garcia-Lopez"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.07372",
     "doi": "10.48550/arXiv.2505.07372"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Quality-filtered synthetic data (threshold 8.5, retaining ~70% of 30K generated samples) significantly outperforms both unfiltered synthetic data and real-world commit data (CommitPackFT) for automated program repair fine-tuning, with ANOVA confirming statistically significant differences (p<0.0001). The best configurations (vulrep_synt_85, vulrep_synt_85_commitpack) achieved 17.18-17.26% Top@1 and 22.47-23.00% Top@5 Perfect Prediction on VulRepair, surpassing VulMaster (20.0%) and VulRepair (16.8%) despite using sampling with 5 candidates rather than beam search with 50. Cross-model LLM evaluation across 6 generator/evaluator models provides a consensus-based quality filtering mechanism that demonstrably improves downstream training effectiveness.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL, GitHub link, or archive is provided anywhere in the paper. No mention of code availability."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The novel synthetic dataset (~30K samples, ~21K filtered) is not released. While VulRepair and CommitPackFT are publicly available existing datasets, the paper's main contribution (the synthetic training data) has no download link or repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions vLLM for inference and the Outlines library for structured output, and lists model names, but provides no requirements.txt, Dockerfile, conda environment file, or detailed environment setup section with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The methodology is described at a conceptual level but not at the level needed to reproduce without significant guesswork."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for all configurations in Table 10 (e.g., vulrep_synt_85 Top@1: 0.0036 std dev). While not confidence intervals, ± std dev over 50 runs provides uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive statistical testing: ANOVA (F=1735.92, p<0.0001 for Top@1; F=646.27, p<0.0001 for Top@5) followed by Tukey's HSD post-hoc pairwise comparisons (Section 5.2). Shapiro-Wilk normality test and Levene's homogeneity of variances test also applied."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tukey's HSD tables (Tables 15-16) report mean differences between all configuration pairs (e.g., vulrep_synt_85 vs vulrep: 0.0549, approximately 5.5 percentage points improvement). Raw PP% values with baseline context are provided throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Each experiment is repeated 50 times but no justification is given for why 50 iterations were chosen. No power analysis or discussion of whether 50 runs is sufficient for the claimed statistical precision."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations across 50 runs are reported in Table 10 for all configurations under both Top@1 and Top@5 settings (e.g., vulrep Top@1 std dev: 0.0039)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines: vulrep (VulRepair training data alone), vulrep_commitpack (with real-world commits), and external benchmarks VulMaster (20.0%), VulRepair (16.8%), and prior Mistral-based work (22.04%)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "VulMaster (Zhou et al., 2024) and the authors' own prior work (de-Fitero-Dominguez et al., 2024) are recent. VulRepair (2022) is the standard benchmark in this space. All are appropriate comparisons."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Five training configurations systematically vary components: baseline (vulrep), +filtered synthetic, +CommitPackFT, +both, +unfiltered synthetic. This isolates the effect of synthetic data, quality filtering, and combination with real-world data."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results reported under both Top@1 (single prediction, temperature 0.4) and Top@5 (five predictions, temperature 0.8) evaluation settings, analogous to Pass@1 and Pass@5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated patches. All evaluation is automated via Perfect Prediction (exact match). Human evaluation of patch quality would be relevant since exact match misses semantically correct but syntactically different patches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Results reported on the VulRepair test set, explicitly separated from training data. Section 4.3 states: 'we removed any overlapping samples between the VulRepair training and test datasets to prevent overfitting.' The test dataset was 'left unchanged.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Per-language and per-bug-type breakdowns are provided for synthetic data quality scores (Tables 5-9), but the actual repair evaluation (PP%) in Table 10 is only reported in aggregate per training configuration. No per-vulnerability-type or per-CWE breakdown of repair performance."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative analysis of failed patches. The paper notes that concurrency issues and arithmetic errors receive lower quality scores during generation, but does not analyze specific cases where the repair model failed to generate correct patches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The unfiltered synthetic dataset (vulrep_synt_full) performed significantly worse than the filtered version despite having ~30% more samples. Adding CommitPackFT to filtered synthetic data hurt Top@5 performance (22.47% vs 23.00%, p<0.0001). These demonstrate that more data can be counterproductive."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'statistically significant improvements' confirmed by ANOVA (p<0.0001); 'quality-filtered synthetic dataset outperforming...in certain scenarios' confirmed in Tables 10, 15-16; 'surpassed existing systems despite using a less computationally intensive decoding strategy' confirmed for Top@5 comparisons with appropriate hedging."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims synthetic data 'improves' repair performance. The experimental design — same base model (Qwen 2.5 Coder 7B), same fine-tuning procedure, same evaluation, varying only training data — is a controlled single-variable manipulation adequate for this causal claim. Statistical testing (ANOVA + Tukey HSD) confirms significance."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Synthetic Code Surgery: Repairing Bugs and Vulnerabilities' and conclusion ('potentially transforming approaches to data scarcity across software engineering tasks') frame the contribution much more broadly than what was tested. Evaluation is only on C/C++ vulnerability repair (VulRepair). While Section 4.4 notes 'the performance metrics...specifically reflect the effectiveness of the models in repairing the C/C++ vulnerabilities,' this scoping is not maintained in the title, abstract, or conclusion."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion considers multiple alternative factors: model size vs. architecture/training data differences explaining quality variation, subjective weighting schemes potentially affecting results differently, methodological differences in decoding strategies complicating external comparisons, and the possibility that synthetic data captures similar knowledge to real-world commits."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently frames results in terms of 'Perfect Prediction rates' and 'PP%' — the metric is well-defined (exact match with ground truth) and claims stay at this granularity. The abstract says 'improvements in Perfect Prediction rates' rather than claiming broader 'code quality' or 'developer productivity.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers provided: Llama-3.1-Nemotron-70B-Instruct-HF, Meta-Llama-3.1-70B-Instruct, Qwen2.5-72B-Instruct, Mistral-Small-Instruct-2409, gemma-2-27b-it, Qwen2.5-32B-Instruct, and Qwen 2.5 Coder 7B. These are precise HuggingFace model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt construction conceptually ('instruct the model to generate examples demonstrating specific bugs') and shows input/output format templates with placeholders in Section 4.3, but never provides the actual full prompt text used for generation or evaluation. The reader cannot reconstruct the exact prompts sent to models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Comprehensive hyperparameters reported: generation temperature 0.7, evaluation temperature 0.2, Top@1 temperature 0.4, Top@5 temperature 0.8. LoRA: rank 16, alpha 32, dropout 0.1. Training: learning rate 3e-4, cosine scheduler, batch size 256, max sequence length 4096, 3 epochs."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is straightforward fine-tuning followed by direct inference — no tools, feedback loops, retry logic, or agent workflows."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 documents preprocessing: deduplication of training data, removal of train/test overlaps with VulRepair, and detailed formatting with special tokens (<inst>, <desc>, <file>, <lines>, <le>, <sep>). Section 4.2 documents the quality filtering pipeline with specific threshold (8.5) and sample counts at each stage."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated 'Limitations' or 'Threats to Validity' section exists. Limitations are discussed inline within the Discussion (Section 6) and Conclusion (Section 7), but there is no labeled subsection specifically for limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'the weights assigned to different evaluation criteria...involve some subjective judgment' (Section 4.2), evaluation limited to C/C++ VulRepair despite multi-language generation, differential model capabilities across languages and bug types creating potential training bias, and methodological differences in decoding strategies complicating external comparisons."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 explicitly states: 'the performance metrics reported in our experiments specifically reflect the effectiveness of the models in repairing the C/C++ vulnerabilities characteristic of this benchmark.' Section 4.5: 'our primary goal is not to outperform existing systems, but rather to investigate the feasibility of synthetic data generation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Neither the synthetic dataset (30K generated, 21K filtered) nor the experimental results (50 runs per configuration) are available for download. Only aggregated statistics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The synthetic data generation process is thoroughly described in Section 4.1: 6 LLMs generating 5,000 samples each, random selection of language and bug type, XML-structured output, automated validation. Quality evaluation protocol detailed in Section 4.2 with scoring criteria, weights, and threshold."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (VulRepair, CommitPackFT) and LLM-generated synthetic data."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Full pipeline documented with counts: 6 models × 5,000 samples → 29,646 valid samples after format validation (Table 3 shows per-model validity rates) → quality scoring (mean 8.58, std 0.86) → filtering at 8.5 threshold → 20,832 samples (70% retention). Breakdowns by language, bug type, and model provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements section lists two funding sources: project TIFON (PLEC2023-010251) and project RACO-NLP (SBPLY/23/180225/000063), both through Spanish research program calls."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Universidad de Alcalá, Departamento de Ciencias de la Computación. No affiliation with any of the model providers (Meta, Alibaba/Qwen, Google/Gemma, Mistral, NVIDIA) whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Spanish government research programs (I+D+i and Castilla-La Mancha regional funding), with no financial stake in any particular outcome regarding LLM performance or synthetic data effectiveness."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for Qwen 2.5 Coder 7B (the fine-tuned base model) is not stated. The model was released in 2024 and could have been pre-trained on data including VulRepair examples."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3: 'we removed any overlapping samples between the VulRepair training and test datasets to prevent overfitting.' The paper also references VulMaster's correction of data leakage in the original VulRepair evaluation. However, this addresses only fine-tuning data overlap, not pre-training contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VulRepair (derived from CVEFixes 2021 and BigVul 2020) was publicly available well before Qwen 2.5 Coder's training. The base model may have seen VulRepair test examples during pre-training. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a computational experiment involving LLM fine-tuning and automated benchmark evaluation."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study generates and evaluates synthetic code data using LLMs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed reported. The paper mentions using vLLM for efficiency and sampling with 5 candidates (vs 50-beam search), but does not quantify actual cost per example or total inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, hardware specifications, or total compute budget stated. Generating ~30K samples with 6 large models (27B-72B parameters) and fine-tuning 5 configurations of a 7B model represents substantial compute that is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Each experiment repeated 50 times with results reported as mean ± std dev (Table 10). The stochasticity arises from sampling-based decoding (temperatures 0.4 and 0.8), effectively testing seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 explicitly states: 'we repeat each experiment 50 times to ensure that the results remain stable and are not affected by chance.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4 mentions 'Our internal testing confirmed that these settings provided a stable baseline, as minor adjustments to these parameters did not result in significant performance variations,' implying some search was done but the budget (configurations tried, compute spent) is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 5 training configurations are reported with full results (Table 10) and statistically compared pairwise (Tables 15-16). No selective reporting — all configurations shown including underperforming ones."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": true,
         "justification": "Tukey's HSD test is used for post-hoc pairwise comparisons, which explicitly controls for family-wise error rate. Section 5.2.3: 'This test systematically evaluates all possible comparisons while controlling for the family-wise error rate.'"
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all training configurations themselves and evaluate against their own baseline. While internal comparisons use the same model/procedure (reducing bias), they do not explicitly acknowledge or discuss author-evaluation bias, particularly for the external comparisons."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis. The paper notes that sampling with 5 candidates is less expensive than beam search with 50, but does not quantify the compute difference or report performance as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Perfect Prediction (exact match) without discussing whether it validly measures repair capability. Semantically correct patches with different syntax would be counted as failures. No discussion of VulRepair's construct validity as a benchmark for vulnerability repair."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The approach is direct fine-tuning and inference without agentic workflows or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "VulRepair data (CVEFixes 2021, BigVul 2020) predates Qwen 2.5 Coder's release (2024). The base model may have been pre-trained on data that includes VulRepair examples or their source code. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The input format provides error descriptions, vulnerable line numbers, and full source code. In real-world use, a developer may not have precise vulnerability line numbers or structured error descriptions. This potential feature leakage from the evaluation setup is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether VulRepair test examples are independent from each other (e.g., from the same repositories, same CVEs, or structurally similar vulnerabilities that could inflate performance)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "While train/test overlap was removed by deduplication for the fine-tuning data, no concrete leakage detection method (canary strings, membership inference, n-gram overlap analysis) was applied to assess pre-training contamination of the base model."
       }
     }
   },
   "claims": [
     {
       "claim": "Quality-filtered synthetic data (threshold 8.5) significantly outperforms unfiltered synthetic data for APR fine-tuning, with ~2 percentage point improvement in both Top@1 and Top@5.",
       "evidence": "Table 10: vulrep_synt_85 achieves 17.18% Top@1 vs vulrep_synt_full 15.21%; 23.00% Top@5 vs 21.13%. Tukey HSD confirms significance (mean diff 0.0197, p<0.0001 for Top@1). Section 5.2.",
       "supported": "strong"
     },
     {
       "claim": "Filtered synthetic data outperforms real-world commit data (CommitPackFT) as a training augmentation strategy.",
       "evidence": "Table 10: vulrep_synt_85 (17.18% Top@1, 23.00% Top@5) vs vulrep_commitpack (16.14% Top@1, 21.79% Top@5). Tukey HSD: mean diff 0.0104 (Top@1, p<0.001) and 0.0121 (Top@5, p<0.001). Tables 15-16.",
       "supported": "strong"
     },
     {
       "claim": "The best configurations surpass VulMaster (20.0%) and VulRepair (16.8%) on the VulRepair test set despite using sampling with 5 candidates rather than beam search with 50.",
       "evidence": "Table 10 and Figure 6: vulrep_synt_85 achieves 23.00% Top@5, exceeding VulMaster 20.0% and VulRepair 16.8%. Authors acknowledge the methodological difference in decoding strategy (Section 5.1, 6).",
       "supported": "moderate"
     },
     {
       "claim": "Cross-model LLM evaluation provides a robust quality assessment framework that mitigates individual model biases.",
       "evidence": "Table 4 shows evaluator variation: Mistral gives highest average (9.27), Nemotron gives lowest (8.09). The cross-model approach averages across these biases. Section 4.2.",
       "supported": "moderate"
     },
     {
       "claim": "Data quality is more important than quantity for training effective APR systems.",
       "evidence": "Filtered dataset (20,832 samples, ~70% of original) significantly outperforms unfiltered (29,646 samples) across all metrics and evaluation settings. Tukey HSD p<0.0001 for all comparisons. Tables 10, 15-16.",
       "supported": "strong"
     },
     {
       "claim": "High-quality synthetic data promotes greater diversity in patch generation, particularly visible in Top@5 results.",
       "evidence": "In Top@5, vulrep_synt_85 (23.00%) significantly outperforms vulrep_synt_85_commitpack (22.47%, p<0.0001), while in Top@1 they are not significantly different (p=0.806). Section 6 interprets this as synthetic data promoting greater solution diversity.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No code or data released",
       "detail": "Neither the synthetic dataset (~21K filtered samples), the generation/evaluation code, nor the fine-tuned models are released. This severely limits reproducibility despite the paper's emphasis on methodological rigor."
     },
     {
       "flag": "Self-evaluation circularity",
       "detail": "LLMs both generate and evaluate the synthetic data with no external ground truth validation. The quality scores are based on LLM judgment of LLM output. While cross-model evaluation mitigates individual bias, there is no independent validation that high LLM-assigned scores correlate with actual training utility."
     },
     {
       "flag": "Evaluation scope narrower than framing",
       "detail": "Synthetic data spans 12 languages and 13 bug types, but evaluation is only on C/C++ vulnerabilities (VulRepair). The title and conclusion frame contributions broadly ('repairing bugs and vulnerabilities,' 'transforming approaches to data scarcity across software engineering tasks') without this qualification."
     },
     {
       "flag": "Non-comparable external baselines",
       "detail": "External comparisons with VulMaster and VulRepair use fundamentally different decoding strategies (beam search with 50 beams vs sampling with 5 candidates). While the authors acknowledge this, the comparison is still prominently featured as evidence of superiority."
     },
     {
       "flag": "Pre-training contamination unaddressed",
       "detail": "Qwen 2.5 Coder 7B (released 2024) was likely pre-trained on data including VulRepair examples (published 2020-2022). Since all configurations use the same base model, relative comparisons remain valid, but absolute PP% numbers may be inflated."
     }
   ],
   "cited_papers": [
     {
       "title": "VulRepair: A T5-based automated software vulnerability repair",
       "authors": ["M. Fu", "C. Tantithamthavorn", "T. Le", "V. Nguyen", "D. Phung"],
       "year": 2022,
       "doi": "10.1145/3540250.3549098",
       "relevance": "Primary evaluation benchmark; fine-tunes CodeT5 for vulnerability repair and provides the dataset used in this paper's experiments."
     },
     {
       "title": "Multi-LLM Collaboration + Data-Centric Innovation = 2x Better Vulnerability Repair",
       "authors": ["X. Zhou", "K. Kim", "B. Xu", "D. Han", "D. Lo"],
       "year": 2024,
       "arxiv_id": "2401.15459",
       "relevance": "VulMaster is a key comparison baseline; also identified and corrected data leakage in the original VulRepair evaluation."
     },
     {
       "title": "Enhanced automated code vulnerability repair using large language models",
       "authors": ["D. de-Fitero-Dominguez", "E. Garcia-Lopez", "A. Garcia-Cabot", "J.-J. Martinez-Herraiz"],
       "year": 2024,
       "doi": "10.1016/j.engappai.2024.109291",
       "relevance": "Authors' prior work on LLM-based APR using Mistral, provides the data formatting approach extended in this paper and serves as an external comparison benchmark (22.04% Top@5)."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00129",
       "relevance": "Comprehensive study of LLMs for APR, directly relevant to understanding the landscape of learning-based program repair approaches."
     },
     {
       "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Xie", "Y. Ma", "W. Sun", "Y. Yang", "Z. Chen"],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Survey of LLM-based APR systems, provides taxonomy of APR approaches and identifies key challenges in the field."
     },
     {
       "title": "Conversational Automated Program Repair",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2301.13246",
       "relevance": "ChatRepair uses conversational feedback loops for zero-shot APR, representing a different paradigm (prompt-based) compared to fine-tuning approaches."
     },
     {
       "title": "ContrastRepair: Enhancing Conversation-Based Automated Program Repair via Contrastive Test Case Pairs",
       "authors": ["J. Kong", "M. Cheng", "X. Xie", "S. Liu", "X. Du", "Q. Guo"],
       "year": 2024,
       "arxiv_id": "2403.01971",
       "relevance": "Integrates positive feedback from passing tests into conversational APR, complementing ChatRepair's approach to LLM-based repair."
     },
     {
       "title": "OctoPack: Instruction Tuning Code Large Language Models",
       "authors": ["N. Muennighoff", "Q. Liu", "A. Zebaze", "Q. Zheng", "B. Hui", "T. Y. Zhuo", "S. Singh", "X. Tang", "L. von Werra", "S. Longpre"],
       "year": 2023,
       "arxiv_id": "2308.07124",
       "relevance": "CommitPackFT dataset used as a baseline for real-world code change data in this paper's experiments."
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis", "Z. Allen-Zhu", "Y. Li", "S. Wang", "L. Wang", "W. Chen"],
       "year": 2021,
       "arxiv_id": "2106.09685",
       "relevance": "LoRA is the fine-tuning method used for all experiments in this paper."
     },
     {
       "title": "Large Language Model for Vulnerability Detection and Repair: Literature Review and the Road Ahead",
       "authors": ["X. Zhou", "S. Cao", "X. Sun", "D. Lo"],
       "year": 2024,
       "arxiv_id": "2404.02525",
       "relevance": "Survey of LLMs for vulnerability detection and repair, directly relevant to the vulnerability repair task studied in this paper."
     },
     {
       "title": "Evaluating Language Models as Synthetic Data Generators",
       "authors": ["S. Kim", "J. Suk", "X. Yue", "V. Viswanathan", "S. Lee", "Y. Wang", "K. Gashteovski", "C. Lawrence", "S. Welleck", "G. Neubig"],
       "year": 2024,
       "arxiv_id": "2412.03679",
       "relevance": "Evaluates LLMs as synthetic data generators, directly related to the core methodology of using LLMs to generate training data."
     },
     {
       "title": "Self-Instruct: Aligning Language Models with Self-Generated Instructions",
       "authors": ["Y. Wang", "Y. Kordi", "S. Mishra", "A. Liu", "N. A. Smith", "D. Khashabi", "H. Hajishirzi"],
       "year": 2023,
       "arxiv_id": "2212.10560",
       "relevance": "Establishes the self-bootstrapping paradigm where LLMs generate their own training data, conceptually related to this paper's synthetic data approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The synthetic data generation approach is practically useful for APR researchers, but requires running 6 large models (27B-72B parameters) and substantial infrastructure."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Quality-filtered synthetic data outperforming real-world commits is mildly surprising, but the quality-over-quantity finding aligns with recent trends in ML data curation."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security risk angle; the paper is about improving automated bug fixing."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims — straightforward empirical methodology paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or synthetic dataset released for others to try."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors from Universidad de Alcalá; no famous lab or product association."
     }
   }
 }