ai-research-survey

scan.json (32585B)

---

 {
   "paper": {
     "title": "Empirical Evaluation of Large Language Models in Automated Program Repair",
     "authors": [
       "Jiajun Sun",
       "Fengjie Li",
       "Xinzhu Qi",
       "Hongyu Zhang",
       "Jiajun Jiang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2506.13186",
     "doi": "10.48550/arXiv.2506.13186"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodeLlama-7B (code-specialized, fine-tuned) consistently outperforms the larger general-purpose LLaMA-2-13B on APR benchmarks, demonstrating that domain adaptation matters more than parameter scale. Correct patches overwhelmingly appear in the first 30 generations across all models, suggesting practical strategies for reducing computational cost. Prompt design significantly affects repair rates—adding repair examples improves all models, while integrating LLM-generated bug analysis helps weaker models but can hurt stronger models due to inaccurate diagnostics. LLMs perform substantially better on algorithmic assignment bugs than enterprise-grade project bugs, and better on Java than C/C++.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we release all generated patches, evaluation scripts, prompt templates, and dataset configurations at our homepage' but provides no URL or repository link anywhere in the paper text. A vague reference to a 'homepage' without an actual link does not count as released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All six evaluation benchmarks (Defects4J, BugsCpp, IntroClass, IntroClass-Java, ConDefects-Java, ConDefects-Py) are publicly available datasets with citations. The authors also claim to release generated patches, though no URL is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, Docker files, or hardware details are provided. The paper does not describe what GPU hardware was used or any software environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. While the paper describes the experimental methodology, there are no commands, scripts, or README-level instructions that would allow replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables IV, V, and VI are reported as point estimates (e.g., '15.7%' repair rate) with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'DeepSeek-Coder achieves the best repair performance') based solely on comparing raw numbers without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., 'CodeLlama's repair count increases by 206.7%', 'LLaMA's correct repairs rising from 1 to 32 (a 3100% increase)', and absolute repair rates across all conditions, providing sufficient magnitude context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why these specific benchmark sizes are adequate for the claims made. No power analysis or discussion of whether, e.g., 106 bugs in BugsCpp is sufficient for reliable conclusions."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single experimental runs with no assessment of result stability across seeds or repeated experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four LLMs are compared against each other across all datasets, and multiple prompt settings (zero-shot, one-shot, two-shot, analysis-augmented) serve as baseline comparisons."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models (CodeLlama 2023, LLaMA-2 2023, StarCoder 2023, DeepSeek-Coder 2024) are contemporary and widely used in APR research. However, notable omissions include proprietary models like GPT-4 and Claude which had been used in recent APR work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 systematically ablates prompt components: zero-shot vs one-shot (removing the example), and the impact of adding bug analysis. These controlled comparisons isolate the effect of prompt design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: C/P (correct/plausible count), Repair Rate (RRate), and Precision. These capture different aspects of repair effectiveness."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section III-E states 'All plausible patches are then manually inspected by the first two authors. If a patch is semantically equivalent to the ground truth patch, it is classified as a correct patch.' This two-stage validation (automated test suite + manual inspection) is a strength."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The authors use established benchmarks without any tuning. The reduction from 200 to 30 patches was informed by preliminary results on Defects4J/BugsCpp and then applied to separate algorithmic datasets, maintaining separation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Tables IV, V, VI), programming language (C vs Java vs Python comparisons), model, prompt setting, bug length (Figure 5), and repair action types (Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows a specific failure case on BugsCpp with analysis of why LLMs failed (bug length). Figure 7 shows how incorrect bug analysis misleads models. Section IV-A2 discusses compilation failures (20%) and incorrect location modifications (76%) on DeepSeek-Coder."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: all LLMs fail badly on BugsCpp (3.5% average RRate), bug analysis integration hurts DeepSeek-Coder (drops 46.6% on ConDefects-Java), and LLaMA shows very poor performance across most benchmarks."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about CodeLlama outperforming LLaMA (Tables IV, V), diminishing returns from model size (Tables IV, V), early correct patches (Figure 4), and prompt design impact (Table VI) are all supported by corresponding experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'fine-tuning on code-related tasks significantly enhances LLMs' repair capabilities' (Finding 2) by comparing CodeLlama-7B vs LLaMA-2-13B. This conflates fine-tuning with parameter count differences (7B vs 13B) and potentially different base model training. The prompt ablation studies (RQ4) are better controlled but still lack formal causal identification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Empirical Evaluation of Large Language Models in Automated Program Repair' implies broad coverage of LLMs, but the study tests only 4 open-source models (7B-33B) with no proprietary models (GPT-4, Claude). The abstract claims about 'modern, large-scale LLMs' are not well-bounded to the tested subset."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A discusses data leakage as an alternative explanation for model performance. RQ3 investigates bug length as a confounding factor for cross-language performance differences. The analysis of incorrect bug analysis (Figure 7) provides alternative explanations for the negative effect on stronger models."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures repair rate and precision directly on benchmark bugs and does not overclaim these as broader metrics like 'software quality' or 'developer productivity.' Claims match the measurement granularity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table II specifies CodeLlama-7B, LLaMA-2-13B, StarCoderBase (15.5B), and DeepSeek-Coder-33B-instruct with providers and years. These are specific enough to identify exact model weights for open-source models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the full prompt structure including the actual guidance text ('// Provide a fix for the buggy function', 'You are a code analysis tool...'), the GCD example with actual code, and the template structure. Table III summarizes all four prompt configurations used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters are reported — temperature, top-p, top-k, max tokens, and other sampling parameters are not mentioned anywhere in the paper, despite generating over 600,000 patches."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is direct LLM inference with prompts — no tool use, retry logic, feedback loops, or multi-step reasoning pipelines."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B documents dataset filtering: selecting single-function bugs from Defects4J (255+228), BugsCpp (106), IntroClass (297 each), and ConDefects (563 each after filtering for cross-language assignment overlap and random selection of one submission per language)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B 'Limitation' and Section V-C 'Threats to Validity' (with Internal and External subsections) provide substantive discussion of study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: only 4 models tested with parameter sizes up to 33B while larger models exist; manual verification by two authors as a potential internal validity concern; datasets may not represent real-world bug complexity; and limited programming language coverage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B explicitly states: 'the capabilities of even larger or more recent models remain unexplored', 'real-world bugs may be more complex than those in the datasets', and 'additional programming languages not included in this study may pose unique challenges.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to release 'all generated patches, evaluation scripts, prompt templates, and dataset configurations' but provides no URL. Without an accessible link, the raw data cannot be verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B describes dataset selection in detail: sources, years, languages, number of bugs, and filtering criteria (single-function bugs, cross-language overlap for ConDefects, random selection of one submission per assignment)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (Defects4J, BugsCpp, IntroClass, ConDefects)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: benchmark selection → single-function bug filtering (with counts: 255, 228, 106, 297, 297, 563, 563) → patch generation (200 or 30 per bug) → deduplication → test suite validation → manual inspection of plausible patches → classification as correct or not."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Tianjin University, University of Electronic Science and Technology of China, and Chongqing University. They evaluate third-party open-source models, not their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The authors evaluate open-source models from Meta, BigCode, and DeepSeek — no apparent conflict, but the lack of any funding disclosure is a gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models. This is critical since Defects4J (2014) and IntroClass (2015) have been public for years before these models were trained."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section V-A acknowledges the risk that 'some of the buggy or fixed code used in selected benchmarks may partially or fully exist in the model's training data' but performs no actual analysis of train/test overlap (no membership inference, n-gram checks, or temporal analysis)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J (2014), IntroClass (2015), and QuixBugs-derived benchmarks have been publicly available on GitHub for years before the 2023-2024 models were trained. The paper's only mitigation is using 'multiple datasets that differ in collection time' without concretely analyzing contamination risk for specific benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation study using LLMs on bug datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Manual patch verification by authors does not constitute a human subjects study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating over 600,000 patches and discussing computational cost as a key theme (RQ2), no actual inference costs (time, GPU hours, tokens, or monetary cost) are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU type, GPU hours, or hardware specifications are mentioned. The paper discusses reducing patches from 200 to 30 for efficiency but never quantifies the actual compute consumed."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds, seed sensitivity, or results across multiple seeds. All results appear to be from single experimental runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states the number of patches generated per bug: 200 for Defects4J and BugsCpp (Section III-E), 30 for subsequent experiments (Section IV-B). Each patch generation constitutes a run."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Generation parameters (temperature, etc.) are not even reported, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All four prompt configurations are reported with full results in Table VI. No cherry-picking of configurations — the paper systematically compares all settings and reports both improvements and degradations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, so no multiple comparison correction is applied. The paper makes dozens of comparative claims across models, datasets, and prompt settings without any statistical testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors evaluate existing open-source LLMs rather than their own system, which mitigates the primary self-comparison bias concern. They do not re-implement baselines — they use the same generation pipeline for all models."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While the paper discusses model parameter size vs performance (diminishing returns theme), it does not report actual compute costs. No GPU hours, inference time, or matched-compute comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses six benchmarks without discussing whether they actually measure real-world repair capability. No analysis of construct validity — e.g., whether fixing single-function bugs from student assignments generalizes to real software maintenance."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. All models receive the same direct prompt-based generation pipeline, so there is no scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J (2014) and IntroClass (2015) have been publicly available for nearly a decade before the 2023-2024 models were trained. The paper acknowledges diversity in 'collection time' but does not specifically analyze temporal leakage for older benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section III-E states 'we provided method-level perfect fault localization, eliminating potential confounding factors introduced by localization inaccuracies.' This is explicitly acknowledged as a deliberate design choice to isolate repair capability, not a leaked feature."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training corpora for the evaluated models include code from the same projects as the benchmarks (e.g., Apache Commons projects in Defects4J are widely available on GitHub)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used. Section V-A discusses leakage risk conceptually ('it is plausible that some of the buggy or fixed code...may exist in the model's training data') but applies no detection techniques (no canary strings, membership inference, or n-gram overlap analysis)."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuned CodeLlama-7B consistently outperforms the general-purpose LLaMA-2-13B across all evaluated datasets despite having fewer parameters.",
       "evidence": "Tables IV and V show CodeLlama fixing 40 vs 19 bugs on Defects4J v1.2, 34 vs 18 on v2.0, 5 vs 1 on BugsCpp, and similar advantages across all algorithmic benchmarks (Section IV-A, IV-B).",
       "supported": "strong"
     },
     {
       "claim": "Increasing model parameter size yields diminishing returns in repair effectiveness for code-specialized models.",
       "evidence": "DeepSeek-Coder (33B, 4.71x CodeLlama's parameters) repaired only 4 fewer bugs than CodeLlama on Defects4J v1.2 (44 vs 40). StarCoder (15.5B) achieves competitive performance to DeepSeek-Coder (33B) on Defects4J (Table IV, Section IV-A).",
       "supported": "moderate"
     },
     {
       "claim": "Correct patches are predominantly generated in early iterations — 76.59% of DeepSeek-Coder's correct patches appear within the top 30 candidates.",
       "evidence": "Figure 4(a) shows cumulative distribution of correct patch rankings on Defects4J. For LLaMA and StarCoder, only one correct patch each falls outside the top 30. Figure 4(b) confirms 89.6% of StarCoder's correct patches on IntroClass are within top 30.",
       "supported": "strong"
     },
     {
       "claim": "Providing repair examples in prompts substantially improves repair performance across all LLMs.",
       "evidence": "Table VI shows consistent drops in zero-shot vs one-shot: LLaMA drops 85.7% on ConDefects-Java (7 to 1 fix), CodeLlama drops 44%, StarCoder drops 29.3%. Average RRate falls from 11.5% to 8.9% on ConDefects-Java (Section IV-D1).",
       "supported": "strong"
     },
     {
       "claim": "Incorporating LLM-generated bug analysis helps weaker models but hurts stronger models.",
       "evidence": "Table VI: LLaMA increases from 1 to 32 correct fixes on ConDefects-Java (+3100%), CodeLlama increases 53.6%. But DeepSeek-Coder drops 46.6% (118 to 63). Figure 7 shows an example of incorrect analysis misleading DeepSeek-Coder (Section IV-D2).",
       "supported": "moderate"
     },
     {
       "claim": "LLMs achieve significantly better repair performance on enterprise-grade Java bugs than C/C++ bugs.",
       "evidence": "Table IV: average RRate on Defects4J is 15.1% vs 3.5% on BugsCpp — a 76.5% reduction. Analysis in Section IV-A2 attributes this to longer bug lengths in C/C++ code.",
       "supported": "moderate"
     },
     {
       "claim": "Smaller models exhibit complementary repair capabilities — each model generates unique correct patches not found by other models.",
       "evidence": "Figure 3 shows Venn diagrams: CodeLlama fixes 9 unique bugs on Defects4J v1.2 that no other model fixes, LLaMA contributes 1 unique fix despite being the weakest model overall (Section IV-A3).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical testing",
       "detail": "All comparative claims across 4 models, 6 datasets, and 4 prompt settings are based on raw number comparisons with no significance tests, p-values, or bootstrap confidence intervals. With over 600,000 patches, generation stochasticity could easily account for small observed differences."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Results appear to be from single experimental runs with no variance, standard deviation, or confidence intervals reported. LLM patch generation is inherently stochastic, yet no assessment of result stability is provided."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, top-k, max tokens, and other sampling parameters are never reported despite generating 600,000+ patches. These parameters significantly affect generation diversity and quality."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Defects4J (2014) and IntroClass (2015) have been publicly available on GitHub for nearly a decade. The evaluated models were trained in 2023-2024 on massive code corpora likely including these benchmarks. No contamination detection methods (membership inference, n-gram overlap) are applied."
     },
     {
       "flag": "Confounded fine-tuning comparison",
       "detail": "The claim that 'fine-tuning on code-related tasks significantly enhances repair capabilities' is based on comparing CodeLlama-7B vs LLaMA-2-13B, which confounds fine-tuning with parameter count (7B vs 13B) and potentially different base training. A fair comparison would require CodeLlama-7B vs LLaMA-2-7B."
     },
     {
       "flag": "No compute costs despite cost-effectiveness theme",
       "detail": "The paper identifies 'Cost-Effectiveness Tradeoffs' as a major gap in prior work and discusses reducing patch counts from 200 to 30, yet never reports actual compute costs (GPU type, GPU hours, inference time, or energy consumption)."
     }
   ],
   "cited_papers": [
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Code-specialized LLM derived from LLaMA via fine-tuning on code tasks, one of four models evaluated in this study."
     },
     {
       "title": "LLaMA 2: Open Foundation and Fine-Tuned Chat Models",
       "authors": ["H. Touvron", "L. Martin", "K. Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "General-purpose open-source LLM used as the non-code-specialized baseline in this APR evaluation."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM evaluated for automated program repair capability across multiple benchmarks."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming — the Rise of Code Intelligence",
       "authors": ["D. Guo", "Q. Zhu", "D. Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Largest code-specialized model evaluated (33B parameters), achieving best overall repair performance in this study."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "First study applying LLMs to APR tasks, evaluated earlier-generation models on Defects4J, ManyBugs, and QuixBugs — a key predecessor to this work."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "ChatRepair approach using conversational LLM interaction for APR, demonstrating cost-effective repair with commercial LLMs."
     },
     {
       "title": "ThinkRepair: Self-Directed Automated Program Repair",
       "authors": ["X. Yin", "C. Ni", "S. Wang"],
       "year": 2024,
       "relevance": "LLM-based APR technique using self-directed reasoning, part of the newer generation of LLM APR approaches."
     },
     {
       "title": "How Far Can We Go with Practical Function-Level Program Repair?",
       "authors": ["J. Xiang", "X. Xu", "F. Kong"],
       "year": 2024,
       "arxiv_id": "2404.12833",
       "relevance": "SRepair: function-level APR with LLMs evaluated primarily on Defects4J, a close comparison point for this study."
     },
     {
       "title": "RepairLlama: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["A. Silva", "S. Fang", "M. Monperrus"],
       "year": 2023,
       "arxiv_id": "2312.15698",
       "relevance": "LoRA-based fine-tuning of CodeLlama specifically for APR tasks, directly relevant to the fine-tuning vs general-purpose model comparison."
     },
     {
       "title": "The Plastic Surgery Hypothesis in the Era of Large Language Models",
       "authors": ["C. S. Xia", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "relevance": "FitRepair approach fine-tuning CodeT5 for APR, exploring the hypothesis that repair ingredients exist in nearby code."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier"],
       "year": 2023,
       "relevance": "Empirical study evaluating earlier code language models (CodeBERT, CodeT5) on APR, a predecessor to this work's evaluation of larger LLMs."
     },
     {
       "title": "ConDefects: A New Dataset to Address the Data Leakage Concern for LLM-based Fault Localization and Program Repair",
       "authors": ["Y. Wu", "Z. Li", "J. M. Zhang"],
       "year": 2023,
       "arxiv_id": "2310.16253",
       "relevance": "Benchmark dataset specifically designed to mitigate data leakage concerns in LLM-based APR evaluation, used as two of the six benchmarks in this study."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable guidance on prompt design and model selection for APR practitioners, plus the finding that 30 patches suffices instead of 200."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that smaller models generate unique patches not found by larger models is mildly surprising, but most results confirm expected trends."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; this is a purely technical evaluation of code repair capabilities."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims — a straightforward empirical comparison."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Claims to release code and patches but provides no URL; nothing is demonstrably available to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates recognizable models (LLaMA, DeepSeek-Coder) but the research group and venue are not high-profile."
     }
   }
 }