ai-research-survey

scan.json (26903B)

---

 {
   "paper": {
     "title": "The Art of Repair: Optimizing Iterative Program Repair with Instruction-Tuned Models",
     "authors": [
       "Fernando Vallecillos Ruiz",
       "Max Hort",
       "Leon Moonen"
     ],
     "year": 2025,
     "venue": "EASE 2025 (29th International Conference on Evaluation and Assessment in Software Engineering)",
     "arxiv_id": "2505.02931",
     "doi": "10.5281/zenodo.15294695"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Data Availability) states: 'The replicability package for this work is available online' with a Zenodo DOI link (https://doi.org/10.5281/zenodo.15294695). The package includes source code, generated patches, and fine-tuned models."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks used (HumanEval-Java and Defects4J) are publicly available. Additionally, the replication package includes generated patches for both benchmarks by all models. The fine-tuning data comes from a public source (Zhu et al. 2021, ref [58])."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using the LUMI supercomputer and eX3 infrastructure but does not provide a requirements.txt, Dockerfile, conda environment, or detailed library versions. No environment specification section is present in the paper text."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states the replication package includes 'the source code required to replicate the experiments presented in this work.' The paper also provides detailed descriptions of the pipeline, prompt templates, strategies, and evaluation metrics in Section 3, which together with the replication package constitute reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Tables 1 and 2 and Figure 4 report only point estimates (number of problems with plausible patches). No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., '78% improvement', 'decreased from 107... to 100') but does not use any statistical significance tests. All comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context. For example, 'the performance of CodeLlama increased the number of fixed problems to 107 (78% improvement)' from a base of 60. Tables 1 and 2 provide absolute numbers with baselines, and Figure 4 shows differences relative to Strategy A."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the benchmark sizes (164 bugs in HumanEval-Java, 217 in Defects4J). No power analysis or discussion of whether these sample sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses beam search (deterministic decoding), so there is only one run per configuration. No variance, standard deviation, or spread measures are reported. While deterministic decoding means identical re-runs would yield the same results, there is no assessment of sensitivity to hyperparameter choices."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes base (non-fine-tuned) models as baselines and compares FFT and LoRA fine-tuning approaches. Strategy A (non-iterative) serves as the baseline for iterative strategies. Results are compared across multiple model configurations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models used (Llama 3.1, DeepSeek-Coder, CodeLlama) are contemporary. The paper compares against and discusses related work including RepairLLaMA (2024), MORepair (2024), and Li et al. (2024). The baselines are recent and relevant."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies: (1) fine-tuning technique (FFT vs LoRA vs base), (2) dataset size (1K, 30K, 65K), and (3) generation strategy (7 strategies varying iterations and outputs). These variations function as ablations showing the contribution of each factor."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses a single primary metric: 'number of problems for which at least one plausible patch is generated.' While position analysis and manual correctness verification add depth, these are not separate evaluation metrics in the traditional sense."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.6 reports: 'we manually inspected 3,298 plausible patches. Of these, 3,167 were confirmed to be correct, while 131 were found to be overfitting to the test suite.' This constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation benchmarks (HumanEval-Java and Defects4J) are separate from the fine-tuning data (GitHub commits from Zhu et al. 2021). The paper explicitly discusses data leakage risk and uses HumanEval-Java specifically because of its recency to reduce contamination risk."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (3 models), fine-tuning approach (base, FFT, LoRA), dataset size (1K, 30K, 65K), benchmark (HumanEval-Java, Defects4J), and strategy (A through G). Tables 1 and 2 and Figure 4 provide detailed per-configuration results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where approaches fail: overfitting with larger datasets, diminishing returns with too many iterations for fine-tuned models, and the reduction in iterative capability after fine-tuning. Section 3.6 reports 131 patches overfitting to tests. The Venn diagrams (Figure 3) show problems only solvable by specific strategies."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: (1) increasing dataset size beyond 1K sometimes decreases performance (Finding 2); (2) fine-tuned models show reduced effectiveness with iterations (Finding 6); (3) Strategy G dropped for Defects4J due to excessive resource demands with diminishing returns; (4) 131 of 3,298 inspected patches were overfitting."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'improvements of up to 78% in the number of plausible patches' — supported by Table 1 (CodeLlama from 60 to 107). Claims about diminishing returns with larger datasets are supported by Tables 1-2. Claims about iterative benefits for base models are supported by Figure 4 and RQ2 results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through controlled single-variable manipulations: varying dataset size while holding other factors constant, varying strategy while holding model constant, etc. The ablation-style design (systematically varying one factor at a time) provides adequate support for claims like 'fine-tuning reduces iterative capability.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java with two benchmarks but states in the conclusion: 'This work does not only advance the field of APR, but it also provides practical guidance for the deployment of LLMs in complex pipelines.' The threats section says 'The insights in this paper should generalize to arbitrary programming languages,' which is an unbounded generalization claim from Java-only experiments."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses multiple alternative explanations for the overfitting phenomenon (Section 4.1.1): data quality, overfitting, and limited model capacity, systematically ruling some out. For fine-tuned models' reduced iterative ability, they discuss over-specialization vs. reduced zero-shot flexibility as competing explanations."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model variants with HuggingFace links: 'meta-llama/Llama-3.1-8B-Instruct' (footnote 1), 'meta-llama/CodeLlama-7b-Instruct-hf' (footnote 3), 'deepseek-ai/deepseek-coder-6.7b-instruct' (footnote 4). These are specific, versioned model identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 provides the exact prompt templates used: the initial repair prompt, the failed-test feedback prompt, and the compilation-error feedback prompt. The templates include the actual text with placeholder variables clearly defined."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using 'beam-based search decoding strategy without stochastic sampling' but does not report specific hyperparameters such as beam width, temperature, max tokens, learning rate for fine-tuning, number of training epochs, batch size, or LoRA rank/alpha. The threats section mentions 'standardized hyperparameters' but does not list them."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 describes the iterative pipeline in detail: how inputs are formatted with bug delimiters, how outputs are parsed, the validation phase (plausible/wrong/timeout/uncompilable), feedback extraction (failed test code or compilation errors), and how feedback is incorporated into subsequent iterations. Section 3.5 details the seven generation strategies."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the fine-tuning data source (143,666 samples from GitHub Java projects, ref [58]), the subset creation (1K, 30K, 65K samples), and the benchmark selection (217 single-hunk bugs from Defects4J v2.0, 164 bugs from HumanEval-Java). Section 3.4 explains how inputs are formatted with bug delimiters."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 is a dedicated 'Threats to validity' section discussing internal, external, and construct validity threats in substantive detail."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: data leakage from Defects4J into pretraining data (mitigated by using HumanEval-Java), hyperparameter selection bias, limitation to Java benchmarks, plausibility vs. correctness gap (with 131/3,298 overfitting patches quantified), and the stopping criteria at 10 patches."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats section mentions Java-only benchmarks, the paper does not explicitly state what the results do NOT show. The conclusion makes broad claims ('provides practical guidance for the deployment of LLMs in complex pipelines') without explicitly bounding the scope. No explicit list of what was not tested or what claims are not being made."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states the replication package includes 'the generated patches for both benchmarks by all models used.' Section 3.6 confirms that 'all generated patch files, the code including the seed used to randomly sample these patches, and manual assessments are released in our replication package.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the fine-tuning data source (Zhu et al. 2021, 143,666 single-hunk fixes from GitHub Java projects) and the benchmarks (Defects4J v2.0 with 217 single-hunk bugs, HumanEval-Java with 164 bugs). The data sources and their characteristics are clearly documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study uses publicly available benchmarks and models. NA."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 documents the full pipeline from input (buggy functions with delimiters) through prompting, output parsing, validation (compile/test), feedback extraction, and iterative refinement. The pipeline steps are described in sufficient detail with specific templates and outcome categories."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding from the Research Council of Norway (secureIT project, IKTPLUSS #288787), the European Union Horizon Europe Marie Sklodowska-Curie Actions (#101151798), and computing resources from eX3 (#270053) and LUMI supercomputer via Sigma2."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with Simula Research Laboratory, Oslo, Norway, as stated in the paper header. They are not evaluating their own company's products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from the Research Council of Norway and EU Horizon Europe, which are public funding bodies with no financial stake in the outcome of APR research. The evaluated models (Llama, CodeLlama, DeepSeek-Coder) are not from the authors' institution or funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The absence of such a disclosure is noted; there is no section addressing potential conflicts."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the three models used (Llama 3.1, CodeLlama, DeepSeek-Coder). This is relevant because Defects4J has been publicly available since 2014."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly discusses this: 'One key internal threat is the potential data leakage from the benchmarks into the pretraining data of the models, specially for older widely-known benchmarks like Defects4J.' They mitigate by using HumanEval-Java and cite Ramos et al. (2025) on LLM memorization of bug benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses contamination: HumanEval-Java is used because of 'its recency, it reduces the risk of data leakage in the pre-training' (Section 3.3). The threats section discusses this risk for Defects4J and cites work showing Llama3.1 has 'less susceptibility to memorization' (ref [62])."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The manual inspection of patches is a verification step, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses GPU memory requirements qualitatively ('up to 200GB of GPU memory after repeated iterations') but does not report inference cost, wall-clock time, tokens consumed, or cost per bug. This is notable since the paper motivates developer-centric efficiency."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using the LUMI supercomputer and eX3 infrastructure but does not state total GPU hours, training time, or total computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuning with only 1K samples can improve plausible patch generation by up to 78% over base models.",
       "evidence": "Table 1 shows CodeLlama FFT(1K) generates plausible patches for 107 problems vs. 60 for the base model on HumanEval-Java (Section 4.1.1).",
       "supported": "strong"
     },
     {
       "claim": "Increasing fine-tuning dataset size beyond certain thresholds leads to diminishing or declining performance, likely due to overfitting.",
       "evidence": "Tables 1-2 show CodeLlama FFT decreasing from 107 (1K) to 100 (65K) on HumanEval-Java, and DeepSeek-Coder from 129 (1K) to 122 (65K). Similar patterns on Defects4J (Section 4.1.1).",
       "supported": "strong"
     },
     {
       "claim": "Base models consistently benefit from iterative feedback strategies while fine-tuned models show reduced effectiveness with iterations.",
       "evidence": "Figure 4 shows base models gaining +4 to +46 problems with iterative strategies, while fine-tuned models lose up to -34 problems with high iteration counts (Section 4.2).",
       "supported": "strong"
     },
     {
       "claim": "There are unique problems that only iterative base models can solve and that fine-tuned models cannot.",
       "evidence": "Figure 3 Venn diagrams show 10 unique problems (12% of solutions) on HumanEval-Java and 26 unique problems (19%) on Defects4J solvable only by iterative base models (Section 4.2.1).",
       "supported": "strong"
     },
     {
       "claim": "Complex problems benefit more from iterative refinement than simpler problems.",
       "evidence": "Defects4J (more complex) shows larger gains from iterative strategies than HumanEval-Java. For example, Llama3.1 Base goes from 28 (Strategy A) to 74 (Strategy F) on Defects4J — a 164% increase (Section 4.2.2).",
       "supported": "strong"
     },
     {
       "claim": "The insights generalize to arbitrary programming languages.",
       "evidence": "Section 5 (Threats to validity) asserts this but experiments are conducted only on Java benchmarks. No cross-language evaluation is provided.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper shows that fine-tuning instruction-tuned LLMs with as few as 1K samples can improve automatic program repair by up to 78%, but larger datasets often yield diminishing returns due to overfitting. Base models benefit substantially from iterative feedback strategies, while fine-tuned models perform best with fewer iterations on simpler tasks. Complex benchmarks (Defects4J) amplify the benefits of iterative refinement. The study identifies model-specific optimal strategies and reveals that some problems are uniquely solvable only through iterative base-model approaches.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims are made by comparing raw numbers without statistical tests. With stochastic elements in the pipeline (despite beam search, the selection of subsets for fine-tuning, benchmark characteristics), the absence of significance tests makes it hard to distinguish signal from noise in the differences reported."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "Results are single-point estimates with no confidence intervals or error bars. The use of deterministic beam search reduces but does not eliminate concerns — different random seeds for fine-tuning data subsets could yield different results."
     },
     {
       "flag": "Missing hyperparameter details",
       "detail": "Critical hyperparameters like beam width, learning rate, number of training epochs, batch size, and LoRA rank/alpha are not reported in the paper. The threats section mentions 'standardized hyperparameters' but the actual values are absent."
     },
     {
       "flag": "Unbounded generalization claims",
       "detail": "The paper claims insights 'should generalize to arbitrary programming languages' and provides 'practical guidance for the deployment of LLMs in complex pipelines' despite testing only on Java with two benchmarks."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Foundational LLM-based APR study at ICSE, directly relevant to evaluating LLM capabilities for code repair."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["I. Bouzenia", "P. Devanbu", "M. Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Agentic APR approach using LLMs with tool integration, directly relevant to agentic AI for software engineering."
     },
     {
       "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["A. Silva", "S. Fang", "M. Monperrus"],
       "year": 2024,
       "arxiv_id": "2312.15698",
       "relevance": "LoRA-based fine-tuning for APR showing PEFT outperforming full fine-tuning, relevant to LLM adaptation for code tasks."
     },
     {
       "title": "A Comprehensive Evaluation of Parameter-Efficient Fine-Tuning on Automated Program Repair",
       "authors": ["G. Li", "C. Zhi", "J. Chen", "J. Han", "S. Deng"],
       "year": 2024,
       "arxiv_id": "2406.05639",
       "relevance": "Evaluates PEFT techniques for APR, finding diminishing returns with dataset size — directly comparable to this paper's findings."
     },
     {
       "title": "Multi-Objective Fine-Tuning for Enhanced Program Repair with LLMs",
       "authors": ["B. Yang", "H. Tian", "J. Ren"],
       "year": 2024,
       "arxiv_id": "2404.12636",
       "relevance": "MORepair: multi-objective fine-tuning of instruction LLMs for APR, a direct competitor approach."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["A. Madaan"],
       "year": 2023,
       "relevance": "Foundational work on LLM self-refinement through iterative feedback, central to the iterative repair paradigm studied here."
     },
     {
       "title": "Conversational Automated Program Repair",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2301.13246",
       "relevance": "Pioneered conversational APR with LLMs using test feedback, directly motivating this paper's iterative approach."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Reports cost-effective conversational APR with ChatGPT, relevant to both LLM capability evaluation and cost-effectiveness."
     },
     {
       "title": "CigaR: Cost-efficient Program Repair with LLMs",
       "authors": ["D. Hidvégi", "K. Etemadi", "S. Bobadilla", "M. Monperrus"],
       "year": 2024,
       "arxiv_id": "2402.06598",
       "relevance": "Cost-efficient LLM-based APR, relevant to practical deployment constraints of AI programming tools."
     },
     {
       "title": "Are Large Language Models Memorizing Bug Benchmarks?",
       "authors": ["D. Ramos", "C. Mamede", "K. Jain"],
       "year": 2025,
       "arxiv_id": "2411.13323",
       "relevance": "Directly addresses benchmark contamination in LLM-based APR, a key methodological concern for code generation evaluations."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["X. Chen", "M. Lin", "N. Schärli", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Self-debugging paradigm for LLMs using execution feedback, foundational for iterative code repair approaches."
     },
     {
       "title": "RLEF: Grounding Code LLMs in Execution Feedback with Reinforcement Learning",
       "authors": ["J. Gehring", "K. Zheng", "J. Copet"],
       "year": 2024,
       "relevance": "Uses reinforcement learning with execution feedback to train code LLMs, relevant to training approaches for code generation."
     }
   ]
 }