ai-research-survey

scan.json (26175B)

---

 {
   "paper": {
     "title": "Enhancing Automated Program Repair through Fine-tuning and Prompt Engineering",
     "authors": [
       "Rishov Paul",
       "Md. Mohib Hossain",
       "Mohammed Latif Siddiq",
       "Masum Hasan",
       "Anindya Iqbal",
       "Joanna C. S. Santos"
     ],
     "year": 2023,
     "arxiv_id": "2304.07840",
     "doi": "10.5281/zenodo.8122636"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a replication package at https://doi.org/10.5281/zenodo.8122636 (footnote 1), described as containing 'all the scripts used to gather the data and results'."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The datasets used (Tufano et al. and Review4Repair) are from publicly available prior work. The replication package on Zenodo is stated to contain all data and scripts."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an NVIDIA GeForce RTX 2070-8GB GPU and names the models used, but does not provide a requirements.txt, Dockerfile, or detailed environment specification listing library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication package is provided on Zenodo, the paper itself does not include step-by-step reproduction instructions, a README description with commands, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates (accuracy percentages, BLEU scores) with no confidence intervals, error bars, or uncertainty measures in Tables II and III."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims fine-tuned models 'notably outperformed' and achieve 'significant margin' improvements but provides no statistical significance tests (no p-values, t-tests, or any formal tests). Comparisons are based solely on numeric differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute improvements with baseline context throughout. For example, Table II shows '+5.69' improvement from 19.59% baseline to 25.28%, and similar formats across all comparisons, providing enough context to understand the magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "For the manual analysis (RQ3), the paper explicitly justifies the sample size: '314 test samples from Tufano et al. and 340 test samples from Review4Repair datasets in order to achieve a 95% confidence interval and 5% error of margin' (Section III-C)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across runs. The paper does not indicate whether experiments were run multiple times, and all results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against baseline models from prior work: R4R CC from Review4Repair and Tufano 2-encoder from Tufano et al. Results are shown in Table II."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines (Review4Repair 2022, Tufano et al. 2021) are the original models from the datasets' source papers, representing the best known results on these specific datasets at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed. The paper does not systematically remove or vary components (e.g., removing code review from input, varying prompt components) to measure individual contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple evaluation metrics: Top-1/5/10 Accuracy (exact match), BLEU-4, and CodeBLEU, as described in Section III-D and reported in Tables II and III."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 includes a manual developer analysis where two software developers scored generated repairs on whether they fulfilled the review intention. Results are in Table IV with Cohen's Kappa inter-rater agreement."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets have explicit train/validation/test splits. The Tufano et al. dataset has 13,756/1,719/1,719 split and Review4Repair has 53,198/2,956/2,955 split (Table I). Results are reported on the test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides per-category breakdowns across three fix categories (Insert, Delete, Update) for both datasets and all models, showing important variation in performance across categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III-B5 discusses common failure patterns in LLM outputs (syntax problems, explanations appended, buggy code mixed in, markdown formatting). The discussion section (V-A) discusses data quality issues in Review4Repair that affect results."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that LLMs are 'still a long way off' for practical APR. RQ3 shows models fulfill only 38-59% of reviews. The abstract and conclusion explicitly state that practical application is not yet feasible. GPT-3.5-Turbo without heuristics had low accuracy (6.9% on Review4Repair)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that fine-tuned models 'notably outperformed' baselines (supported by Table II), that LLMs show promise with prompting (supported by Table III), and that practical application 'is still a long way off' (supported by RQ3 manual analysis in Table IV)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims such as 'the boost in accuracy is due to mostly the learned parameters of the model rather than the architecture itself' (Section VIII) without adequate experimental support. No controlled experiment isolates the contribution of pre-trained parameters vs. architecture. The comparison between PLBART and CodeT5 does not control for architecture since both differ in architecture AND pre-training."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity section (VI) explicitly acknowledges that 'the datasets consisted of only Java codes and respective code reviews in the English language; hence, our focus was confined to a single programming language. As a result, the coverage of our findings is limited.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses data leakage as an alternative explanation for LLM performance (Section VI): 'One possible reason might be that these LLMs were also trained with our aforementioned datasets.' It also discusses data quality issues in the Review4Repair dataset as a confound for model performance in Section V-A."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GPT-3.5-Turbo' and 'Code-DaVinci-Edit-001' without specifying exact API versions or snapshot dates. For PLBART and CodeT5, specific model versions are not given beyond referencing the original papers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Listing 2 provides the full prompt structure for both zero-shot and few-shot scenarios. The system role content is quoted verbatim: 'You are a coding assistant. You generate only the source code.' The user prompt instruction is given: 'Refactor the Buggy Code using the Review without comments.' The Code-DaVinci-Edit-001 instruction is also quoted."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A2 reports hyperparameters for fine-tuning (input length 512, target length 200, beam sizes 1/5/10, epochs 11-12 for PLBART, batch size 4, gradient accumulation 8, 45 epochs for CodeT5). Section III-B4 reports LLM parameters: temperature=0, top_p=1, frequency_penalty=0, presence_penalty=0."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The models are used in a straightforward prompt-response or fine-tuning paradigm without multi-step reasoning, tool use, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A1 describes dataset preprocessing in detail: concatenation of buggy code with review, special tokens used, token length filtering (removing 57+6 samples exceeding 512 tokens), and reorganization of train/val/test splits with specific percentages and counts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides a dedicated discussion of both internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses specific issues: hyperparameter search space limitations, restriction to Java-only datasets, potential data leakage with GPT-3.5-Turbo (training data knowledge cutoff of September 2021 vs. dataset publication dates), and black-box nature preventing verification."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: 'our focus was confined to a single programming language' (Java), datasets from only two sources (Gerrit and GitHub), and the paper acknowledges that 'other datasets of various programming languages might be investigated in future research.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package is available at Zenodo (https://doi.org/10.5281/zenodo.8122636) and the underlying datasets from prior work (Tufano et al. and Review4Repair) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A1 describes data collection: both datasets are from prior published work, consisting of real code reviews collected from Gerrit and GitHub. Dataset sizes and compositions are given in Table I."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For RQ3's manual analysis, the two developers who performed the evaluation are described minimally ('one year of industry experience in a Fortune 500 company') but how they were recruited is not described. The selection process for these specific evaluators is not discussed."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented in Section III-A1: from original datasets to preprocessing (concatenation, token filtering, train/val/test splits) with specific counts at each stage (e.g., 56,211 samples, removing 57+6 exceeding 512 tokens, final 53,198/2,956/2,955 split)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: BUET (Bangladesh), University of Notre Dame (USA), University of Rochester (USA). The paper evaluates OpenAI models but no authors are affiliated with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be verified. The absence of a funding disclosure statement means this criterion cannot be satisfied."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper. Two authors list IQVIA email addresses ({rishov.paul, mdmohib.hossain}@iqvia.com) suggesting industry affiliation, but this potential conflict is not addressed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in Section VI: 'The knowledge cut-off of these two models is September 2021' for GPT-3.5-Turbo and Code-DaVinci-Edit-001."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly discusses data leakage risk: 'One possible reason might be that these LLMs were also trained with our aforementioned datasets. As a result, there might be a data leakage.' It notes the Tufano et al. dataset was published before the cutoff and Review4Repair after it."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses this in Section VI: the Tufano et al. dataset (2021) was published before the September 2021 training cutoff, while Review4Repair (2022) was published after. The paper acknowledges: 'As these models are black-box, there is no way we can verify if there is data leakage for these datasets.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "The study includes human evaluation (RQ3 with two developers scoring 654 samples) but no pre-registration is mentioned."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "The manual analysis involves two human evaluators rating code repairs, but no IRB or ethics approval is mentioned."
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Some demographics are reported for the two evaluators: 'one year of industry experience in a Fortune 500 company and significant involvement in the code review process in software development' (Section III-C)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for selecting the two developers. The paper does not explain why these specific evaluators were chosen or what qualifications were required."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is not an experimental study with treatment and control conditions requiring randomization. The manual analysis has all evaluators scoring the same samples."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe whether evaluators knew which model produced each repair. Given they scored outputs from five named models, it appears they were not blinded, which could bias judgments."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information is given about whether both raters completed all 654 evaluations or if any were dropped. The paper does not discuss attrition or incomplete evaluations."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses OpenAI API models (GPT-3.5-Turbo, Code-DaVinci-Edit-001) across thousands of test samples but does not report API costs, tokens consumed, or inference time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions using an NVIDIA GeForce RTX 2070-8GB GPU, it does not report total training time, GPU hours, or API spend for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "Fine-tuned PLBART and CodeT5 significantly outperform baseline models on both datasets for code repair using code review",
       "evidence": "Table II shows improvements of +5.69% to +25.65% in Top-1/5/10 accuracy over baselines on both Review4Repair and Tufano et al. datasets.",
       "supported": "moderate"
     },
     {
       "claim": "CodeT5 generally performs better than PLBART for code repair, especially for Insert and Update categories",
       "evidence": "Table II and Figure 2 show CodeT5 achieves higher accuracy in most configurations. Per-category breakdowns in Figure 2 confirm CodeT5 superiority in Insert and Update classes.",
       "supported": "moderate"
     },
     {
       "claim": "Zero-shot GPT-3.5-Turbo achieves competitive results after applying heuristic post-processing",
       "evidence": "Table III shows zero-shot GPT-3.5-Turbo with heuristics achieves 22.06% (Review4Repair) and 31.70% (Tufano et al.) accuracy, improving by 15.6% and 12.27% over raw output.",
       "supported": "moderate"
     },
     {
       "claim": "Code-DaVinci-Edit-001 achieves state-of-the-art performance on the Tufano et al. dataset with 40.70% accuracy",
       "evidence": "Table III shows Code-DaVinci-Edit-001 achieving 40.70% accuracy on Tufano et al. dataset, outperforming all other models including fine-tuned ones.",
       "supported": "weak"
     },
     {
       "claim": "Language models still cannot reliably fulfill code review intentions in repairs",
       "evidence": "Table IV (RQ3) shows the best model fulfills only 52.65% (Review4Repair) and 58.92% (Tufano et al.) of reviews according to manual developer analysis.",
       "supported": "strong"
     },
     {
       "claim": "The accuracy boost is mostly due to learned parameters rather than architecture",
       "evidence": "This claim is made in Section VIII (Conclusion) but no controlled experiment isolating pre-trained weights from architecture is presented.",
       "supported": "unsupported"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "Fine-tuned pre-trained models (PLBART, CodeT5) substantially outperform prior baseline models on two code repair datasets when using code review as input, with CodeT5 achieving the best overall performance. Zero-shot and few-shot prompting with GPT-3.5-Turbo and Code-DaVinci-Edit-001 show competitive results, particularly after heuristic post-processing of outputs. However, manual developer analysis reveals that even the best models fulfill only about 50-59% of code review intentions, indicating practical application of LLMs for automated program repair remains premature.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper uses language like 'notably outperformed' and 'significant margin' to describe performance differences but provides no statistical tests (p-values, confidence intervals, or hypothesis tests). All comparisons are based solely on point estimate differences."
     },
     {
       "flag": "No variance or multiple-run reporting",
       "detail": "All results appear to be from single runs. For fine-tuned models with stochastic training, this makes it impossible to know whether observed differences are due to the method or random variation in training."
     },
     {
       "flag": "Potential data leakage for LLM results",
       "detail": "The paper acknowledges that GPT-3.5-Turbo's training data cutoff (September 2021) overlaps with the Tufano et al. dataset (published 2021). Code-DaVinci-Edit-001 achieves suspiciously high accuracy (40.70%) on this dataset, which may reflect memorization rather than genuine program repair capability."
     },
     {
       "flag": "Unsupported causal claim about learned parameters",
       "detail": "The conclusion claims 'this boost in accuracy is due to mostly the learned parameters of the model rather than the architecture itself' without any ablation or controlled experiment to support this attribution."
     },
     {
       "flag": "Only two human evaluators for manual analysis",
       "detail": "RQ3 relies on only two developers for manual evaluation. With Cohen's Kappa ranging from 0.51 to 0.68 (moderate to substantial agreement), the inter-rater reliability is not very high, and the small number of evaluators limits the robustness of conclusions from the manual analysis."
     },
     {
       "flag": "Undisclosed industry affiliations",
       "detail": "Two authors list IQVIA email addresses, suggesting industry affiliation, but no conflicts of interest statement addresses this. No funding sources are disclosed at all."
     }
   ],
   "cited_papers": [
     {
       "title": "Review4Repair: Code Review Aided Automatic Program Repairing",
       "authors": ["F. Huq", "M. Hasan", "M. M. A. Haque", "S. Mahbub", "A. Iqbal", "T. Ahmed"],
       "year": 2022,
       "doi": "10.1016/j.infsof.2021.106765",
       "relevance": "Introduces a key dataset and baseline for code-review-based automated program repair, directly related to evaluating LLM capabilities for code generation."
     },
     {
       "title": "Towards Automating Code Review Activities",
       "authors": ["R. Tufano", "L. Pascarella", "M. Tufano", "D. Poshyvanyk", "G. Bavota"],
       "year": 2021,
       "relevance": "Establishes transformer-based code repair using code review with one-encoder and two-encoder models, a key baseline for LLM-based program repair evaluation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark, foundational for evaluating LLM code generation capabilities."
     },
     {
       "title": "Unified Pre-Training for Program Understanding and Generation",
       "authors": ["W. Ahmad", "S. Chakraborty", "B. Ray", "K.-W. Chang"],
       "year": 2021,
       "relevance": "Introduces PLBART, a key pre-trained model for code understanding and generation tasks evaluated in this study."
     },
     {
       "title": "CodeT5: Identifier-Aware Unified Pre-Trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2021,
       "relevance": "Introduces CodeT5 with identifier-aware pre-training, the best-performing fine-tuned model in this study."
     },
     {
       "title": "GitHub Copilot AI Pair Programmer: Asset or Liability?",
       "authors": ["A. Moradi Dakhel", "V. Majdinasab", "A. Nikanjam", "F. Khomh", "M. C. Desmarais", "Z. M. J. Jiang"],
       "year": 2023,
       "relevance": "Evaluates AI pair programming tools, directly relevant to understanding LLM code generation capabilities and limitations."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Evaluates security of AI-generated code, relevant to assessing quality and safety of LLM code generation."
     },
     {
       "title": "Exploring the Effectiveness of Large Language Models in Generating Unit Tests",
       "authors": ["M. L. Siddiq", "J. C. S. Santos", "R. H. Tanvir", "N. Ulfat", "F. A. Rifat", "V. C. Lopes"],
       "year": 2023,
       "relevance": "Evaluates LLMs for test generation using zero-shot prompting, directly relevant to LLM capabilities in software engineering tasks."
     },
     {
       "title": "Can OpenAI's Codex Fix Bugs?: An Evaluation on QuixBugs",
       "authors": ["J. A. Prenner", "H. Babii", "R. Robbes"],
       "year": 2022,
       "relevance": "Evaluates Codex for automated bug fixing, directly relevant to LLM-based program repair."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": ["N. Carlini", "F. Tramer", "E. Wallace", "M. Jagielski"],
       "year": 2020,
       "relevance": "Addresses data leakage and memorization in LLMs, relevant to benchmark contamination concerns in LLM evaluations."
     },
     {
       "title": "Towards Generating Functionally Correct Code Edits from Natural Language Issue Descriptions",
       "authors": ["S. Fakhoury", "S. Chakraborty", "M. Musuvathi", "S. K. Lahiri"],
       "year": 2023,
       "relevance": "Explores LLM-based code editing from natural language, closely related to prompt-engineering-based program repair."
     }
   ]
 }