ai-research-survey

scan.json (33313B)

---

 {
   "paper": {
     "title": "Exploring Generalizable Automated Program Repair with Large Language Models",
     "authors": [
       "Viola Campos",
       "Ridwan Shariffdeen",
       "Adrian Ulges",
       "Yannic Noller"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.03283"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "No single LLM generalizes well across programming languages for automated program repair; different models excel on different language-specific benchmarks (Java, JavaScript, Python, PHP). Combining models in a committee improves pass@5 by 2–5 percentage points over the best individual model. Adding test case information to prompts yields substantial improvements (up to +47% pass@1), while automated fault localization causes dramatic accuracy drops (7–17% pass@1) compared to perfect localization, with only 28/100 bugs having the correct fix location in the top-3 candidates. Open models are catching up to closed models, with DeepSeek R1 (distilled) surpassing some closed models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'Upon acceptance, we will update the artifact with a reproduction package including the scripts for prompting the LLMs as well for the data analysis.' This is a promise of future release, not a current release. Only results and generated patches are currently available on figshare."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states results and generated patches are 'openly available in our supplemental material' on figshare. Additionally, all four benchmarks (Defects4J, BugsInPy, BugsJS, BugsPHP) are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided. The paper mentions model names and temperature but not the computational environment for running experiments."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. Scripts are promised for future release upon acceptance (Section 7)."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 4, 6, and 7 report pass@k as point estimates without confidence intervals or error bars. No ± notation or CI notation appears anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states: 'we apply the Wilcoxon signed-rank test at a significance level of α = 0.05.' Results tables use bold for best and underline for results not significantly different from the best."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute percentage differences with baselines for context. For example, Table 4 shows base vs. test prompt differences (e.g., 19.02% → 45.33% for Claude 3.7 on Java), and the text reports specific improvements like '+47% pass@1' and '-16.20%' for FL comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "The number of generations per bug (n=15) is justified by citing prior work: 'Based on the standard deviation analysis of pass@1 for LLM-based APR from [27], we use n = 15 as a reasonable, yet manageable number of generations.' The 100-bug subsets are justified through stratified sampling preserving complexity distributions (Section 4.1.2)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as pass@k point estimates computed from 15 generations (3 runs × 5 samples). No standard deviation, IQR, or variance across the 3 independent runs is reported. The reader cannot assess result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "13 LLMs are compared against each other, including both open and closed models. The base prompt serves as a baseline condition, with test and line-level localization prompts as variations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include recent frontier models: Claude 3.7 Sonnet, OpenAI o3-mini, DeepSeek R1, Gemini 2.0 Flash, and Qwen 2.5 Coder. Model selection was based on recent code-focused leaderboards (Section 3.2)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies prompt components: base (code only), test (adding test info), and line-level localization (adding fix location hints). This functions as an ablation of prompt ingredients, with Tables 4 and 6 showing the impact of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are reported throughout: pass@1 (success with one candidate) and pass@5 (success among five candidates), following the Chen et al. formulation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.4 states 'a manual review of patches is infeasible at scale.' Evaluation is entirely automated using test-suite plausibility. No human review of patch quality was conducted."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.5 mentions preliminary experiments to select the prompt format: 'In preliminary experiments, we evaluated different methods for integrating such line-level localization information into APR prompts.' It is unclear whether these preliminary experiments used separate data from the reported results, creating potential data snooping."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by programming language (4 benchmarks), prompt type (base/test/LL), patch complexity (single-line/single-hunk/multi-hunk in Table 7), and model type (open vs. closed). Figure 2 shows per-model unique fix contributions via Venn diagrams."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 analyzes Python indentation errors as a systematic failure mode. Section 4.2.2 discusses cases where automated FL fails to identify the correct function (72/100 bugs). The paper also discusses cases where plausible patches are produced despite incorrect localization (attributed to test overfitting)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: line-level localization hurts accuracy for 4/6 LLMs on PHP (Section 4.2.1); automated FL causes dramatic accuracy drops (Table 6); no single model generalizes across languages; Python performance is surprisingly poor due to indentation issues."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's three key claims are all supported: (1) 'Different LLMs tend to perform best for different languages' — Table 2 shows 4 different top models across 4 benchmarks. (2) 'Combining models by pooling repairs adds value' — Table 5 and Figure 3 show ensemble gains. (3) 'significant drops in accuracy from imperfect FL' — Table 6 shows 7–17% pass@1 drops."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'including information about the failing test case significantly boosted the repair performance' are supported by controlled comparisons where only one prompt component changes while all else remains equal (same models, same bugs, same evaluation). This is a valid single-variable manipulation design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'Our broad selection of models and benchmarks reduces the threat to external validity; however, we cannot claim generality beyond our experiments.' The paper appropriately scopes findings to the tested models, languages, and benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses several alternative explanations: data leakage (with specific leak ratios from Zhou et al.), output variability of LLMs, the distinction between plausibility and correctness (test overfitting), prompt optimization effects, and repair time differences between models."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 explicitly distinguishes between plausible patches (passing tests) and correct patches (satisfying intended requirements): 'A patch may pass all tests but still fail to implement the intended functionality.' They acknowledge using plausibility as a proxy and discuss test overfitting at length."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "All 13 models are specified with identifiable names and sizes: Claude 3.7 Sonnet, Claude 3.5 Haiku, Gemini 1.5 Pro, Gemini 2.0 Flash, GPT-4o (Nov 11 2024 version), o3-mini (Jan 31 2025 version), CodeLlama 13B/70B, DeepSeek Coder 33B, DeepSeek R1 distilled to Llama 70B, Qwen 2.5 Coder 33B, Llama 3.3 70B, CodeGemma 7B. Version dates are given for continuously-updated OpenAI models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Listings 1–4, showing system messages and user prompts with placeholders for benchmark code. The fill values (buggy functions, test cases, error messages) come from the public benchmarks, making prompts fully reconstructible. Supplemental material is also referenced."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states: 'each respective model's standard setting with a temperature of 1.0.' Temperature is the key sampling hyperparameter. Other settings are stated to use defaults."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Section 3.5 states: 'the prompts are not iterative, meaning that each model processes a sample/query without follow-up interactions.' All experiments use single-prompt zero-shot setups."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The filtering pipeline is documented: benchmarks are filtered for reproducibility, single-function bugs, and single-hunk bugs (Table 1 shows counts at each stage). For the 100-bug subsets, stratified sampling by complexity level is described (Section 4.1.2, Table 3), ensuring all repositories are represented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Discussion & Threats to Validity' provides substantial discussion of limitations across multiple paragraphs covering data leakage, LLM variability, prompt optimization, repair time, and plausibility vs. correctness."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: Defects4J has a leaked ratio of 0.41% and BugsInPy 11.0% (citing Zhou et al.); plausibility vs. correctness as a construct validity concern (citing Petke et al.); DeepSeek R1's longer inference time creating unfair comparison; prompt optimization was intentionally avoided to prevent model-specific biases."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: single-function fixes only (excluding multi-function), zero-shot prompts only (excluding iterative/agentic workflows), no prompt optimization was performed, repair time was not analyzed rigorously, and 'we cannot claim generality beyond our experiments' (Section 5)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states: 'The data that support the findings of this study, including all our results and the generated patches, are openly available in our supplemental material: https://figshare.com/s/947fd7030f10a67a1c9f.' This includes raw patch outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes benchmark selection criteria (diverse languages, real bugs, executable with test suites, human ground-truth patches, sufficient reproducible bugs). Patch generation is described: 15 patches per bug via 3 independent runs of 5 generations each at temperature 1.0."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (Defects4J, BugsInPy, BugsJS, BugsPHP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows the filtering pipeline with counts at each stage (all bugs → single-file → single-function → single-hunk → single-line). Section 4.1.2 describes stratified sampling for 100-bug subsets with the distribution shown in Table 3. Reproducibility checks are described per benchmark."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or sponsors are mentioned anywhere in the paper. There is no acknowledgments section listing funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: RheinMain University of Applied Sciences, SonarSource (Singapore), and Ruhr University Bochum. Section 8 provides a disclaimer regarding SonarSource."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making independence impossible to verify. One author is affiliated with SonarSource, a company in the code quality space, which could have a commercial interest in APR outcomes. Section 8 provides a disclaimer but no formal independence statement."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No formal competing interests or financial interests statement is provided. Section 8 has a disclaimer about SonarSource but does not constitute a standard financial interests declaration covering patents, equity, or other commercial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Model version dates are given for GPT-4o (Nov 11 2024) and o3-mini (Jan 31 2025), but these are release dates, not training data cutoff dates. Training cutoff dates are not stated for any of the 13 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly discusses overlap: 'we have to assume that some of the benchmark data may have been included in the training corpora of some of the LLMs.' They cite Zhou et al. showing Defects4J has 0.41% leaked ratio and BugsInPy 11.0%, and Ramos et al. on memorization in open-source models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 addresses contamination by citing Zhou et al.'s LessLeak-Bench analysis of leaked ratios and Ramos et al.'s study of memorization. They note 'BugsInPy obviously has some data leakage issues' yet was the most challenging benchmark, providing an empirical counterpoint to contamination concerns."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of LLMs on code repair tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLM performance on public code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or per-example latencies are reported despite generating approximately 195,000 patches. Section 5 mentions timing differences between models ('spanning between seconds and several minutes') but provides no specific cost data."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No GPU hours, total API spend, or hardware specifications are reported. Section 5 acknowledges: 'Our study methodology did not limit the time for model inference or patch validation.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Three independent runs of 5 generations each are performed, but results are aggregated into a single pass@k estimate. No breakdown by run or analysis of how results vary across the 3 independent runs is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 states: 'We distributed the generation across three independent runs, each generating five candidate patches using each respective model's standard setting with a temperature of 1.0.' Total n=15 per bug."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.5 mentions 'preliminary experiments' to evaluate prompt integration methods for line-level localization, but does not report how many configurations were tried or the compute spent on this search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The LL prompt format was selected from preliminary experiments ('The most effective strategy was found to be a simple comment \"TODO: Fix here:\"') but the paper does not report all configurations tried, what data was used for selection, or the selection criterion beyond effectiveness."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Wilcoxon signed-rank tests are applied across 13 models × 4 languages (52+ comparisons per table) at α = 0.05 with no mention of Bonferroni, Holm, or other multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors do not propose their own APR system; they evaluate 13 third-party LLMs using standardized prompts and public benchmarks. No self-comparison bias exists since they are not comparing their own tool against baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 acknowledges 'Different resource demands across the models, in particular with respect to timing, can lead to unfair comparison' but does not report performance as a function of compute budget. DeepSeek R1 is noted as slower but no quantification is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses construct validity at length: plausibility vs. correctness of patches, test overfitting as a proxy issue, and comparison of metrics (plausibility, TCE, SYE). They cite Petke et al. who found overfitting 'may be less problematic than previously assumed.'"
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models are evaluated using the same zero-shot prompt templates, eliminating scaffold confounds by design."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses temporal leakage: 'Related works, e.g., RepairBench used a dataset with more recent data, whose time period is now also included in the cut-off dates of the latest models.' They cite Zhou et al. on leaked ratios of the specific benchmarks used."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through context. Different prompt variants provide different amounts of information (base vs test vs LL) but this is studied as a variable, not as a leakage concern."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether bugs from the same projects within benchmarks create non-independence. Defects4J contains bugs from 17 projects and BugsInPy from 17 projects, but potential correlations within projects are not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied by the authors. They cite external analyses (Zhou et al.'s leaked ratios, Ramos et al.'s memorization study) but do not apply their own canary strings, membership inference, or decontamination techniques."
       }
     }
   },
   "claims": [
     {
       "claim": "Different LLMs tend to perform best for different programming languages, making cross-platform single-LLM repair difficult.",
       "evidence": "Table 2 shows four different models leading on the four benchmarks: Claude 3.7 Sonnet on Java (22.69% pass@1), Claude 3.5 Haiku on JavaScript (14.48%), DeepSeek R1 distilled on PHP (18.43%), and Gemini 2.0 Flash on Python (12.51%).",
       "supported": "strong"
     },
     {
       "claim": "Including test case information significantly boosts LLM-based repair performance across all models and languages.",
       "evidence": "Table 4 shows consistent improvements with the test prompt, reaching up to +47% pass@1 (DeepSeek R1 on Python). Python shows the strongest average improvement (+34.7% pass@1 averaged over all models). Section 4.1.2.",
       "supported": "strong"
     },
     {
       "claim": "Combining models by pooling repairs adds value, with a committee of expert models outperforming any single model.",
       "evidence": "Table 5 and Figure 3 show ensemble gains: pass@5 improved in 14/16 experiments with the test prompt. Best improvements include JavaScript from 68.00% to 71.68% (o3-mini & DeepSeek R1) and PHP from 58.44% to 63.12%.",
       "supported": "strong"
     },
     {
       "claim": "Automated fault localization leads to significant drops in repair accuracy compared to perfect localization.",
       "evidence": "Table 6 shows pass@1 drops of 7.83–16.20 percentage points when using FLACOCO instead of perfect FL on Defects4J. The actual fix location appeared in the top-3 candidates for only 28/100 faults. Section 4.2.2.",
       "supported": "strong"
     },
     {
       "claim": "Open models are catching up to closed models in APR capabilities.",
       "evidence": "Figure 4 shows a positive trend for open models over time, with DeepSeek R1 (distilled) achieving pass@5 of 25.85% (Table 2), surpassing several closed models. Section 4.4.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs show good robustness when repairing bugs of higher patch complexity.",
       "evidence": "Table 7 shows average pass@1 dropping from 45.06% (single-line) to 34.37% (single-hunk) to 27.50% (multi-hunk). Performance improves from single-line to single-hunk in 9/48 cases and from single-hunk to multi-hunk in 10/48 cases.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Plausibility as proxy for correctness",
       "detail": "The paper uses test-passing (plausibility) as the primary evaluation metric for ~195,000 patches without any manual correctness verification. Test overfitting means patches may pass tests without implementing intended fixes. While acknowledged and discussed (Section 3.4), this inflates apparent fix rates."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Wilcoxon signed-rank tests are applied at α = 0.05 across many model-language-prompt comparisons (potentially hundreds of tests) without any family-wise error rate correction such as Bonferroni or Holm."
     },
     {
       "flag": "No cost or compute budget reported",
       "detail": "Approximately 195,000 patches were generated across 13 models and 4 benchmarks, but no API costs, compute time, or total budget is reported. This limits assessment of practical viability and fairness of model comparisons."
     },
     {
       "flag": "Potential benchmark contamination",
       "detail": "Defects4J and BugsInPy are well-known benchmarks likely present in LLM training data. The paper acknowledges Defects4J has 0.41% and BugsInPy 11.0% leaked ratios, but applies no decontamination or detection method of its own."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the pass@k metric and HumanEval benchmark used for evaluating LLM code generation; foundational for this paper's evaluation methodology."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "doi": "10.1145/3540250.3549101",
       "relevance": "Pioneered zero-shot LLM-based program repair, establishing the approach this paper extends across languages and models."
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 Each using ChatGPT",
       "authors": ["Chunqiu Xia", "Lingming Zhang"],
       "year": 2024,
       "doi": "10.1145/3650212.3680323",
       "relevance": "Demonstrates LLM-based APR with conversational feedback and cost analysis, directly relevant to evaluating LLM repair capabilities."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00129",
       "relevance": "Extensive prior evaluation of 9 LLMs on APR benchmarks across languages, directly compared with this paper's expanded study."
     },
     {
       "title": "RepairBench: Leaderboard of Frontier Models for Program Repair",
       "authors": ["André Silva", "Martin Monperrus"],
       "year": 2024,
       "arxiv_id": "2409.18952",
       "relevance": "Provides a standardized APR leaderboard for frontier models; this paper adopts RepairBench's test prompt template."
     },
     {
       "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["André Silva", "Sen Fang", "Martin Monperrus"],
       "year": 2024,
       "arxiv_id": "2312.15698",
       "relevance": "Explores fine-tuned LLMs for APR with different input/output representations, evaluating systematic prompt and model choices."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar T. Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Demonstrates agentic LLM-based APR with iterative repair cycles, contrasting with this paper's single-prompt approach."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret"],
       "year": 2024,
       "relevance": "Key agentic APR system demonstrating LLM-driven software engineering, related to the paper's discussion of future agentic workflows."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "doi": "10.1145/3650212.3680384",
       "relevance": "Autonomous LLM-based program improvement agent relevant to the paper's discussion of agentic APR approaches."
     },
     {
       "title": "The Fact Selection Problem in LLM-Based Program Repair",
       "authors": ["Nikhil Parasaram", "Huijie Yan", "Boyu Yang"],
       "year": 2024,
       "arxiv_id": "2404.05520",
       "relevance": "Studies how different information in prompts affects LLM-based APR; directly informs this paper's prompt design and the finding that test information is a key signal."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00125",
       "relevance": "Studies fine-tuned code LLMs for APR across benchmarks, providing a baseline comparison for this paper's zero-shot evaluation."
     },
     {
       "title": "LessLeak-Bench: A First Investigation of Data Leakage in LLMs Across 83 Software Engineering Benchmarks",
       "authors": ["Xin Zhou", "Martin Weyssow", "Ratnadira Widyasari"],
       "year": 2025,
       "arxiv_id": "2502.06215",
       "relevance": "Quantifies data leakage rates for SE benchmarks including Defects4J and BugsInPy, directly cited to address contamination threats in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "APR practitioners can use the model comparison insights and model committee strategy, but no tool or code is currently released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that no single LLM dominates across languages is somewhat expected; the dramatic impact of automated FL is noteworthy but not shocking."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No security or safety concerns raised; the paper focuses on bug-fixing capabilities of LLMs."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward empirical comparison study."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No demo, tool, or released code; scripts are promised upon acceptance but not yet available."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From academic labs (RheinMain, Ruhr University) and SonarSource; evaluates well-known models (GPT-4o, Claude, Gemini) but the paper itself is not from a major AI lab."
     }
   }
 }