ai-research-survey

scan.json (24565B)

---

 {
   "paper": {
     "title": "Boosting Redundancy-based Automated Program Repair by Fine-grained Pattern Mining",
     "authors": [
       "Jiajun Jiang",
       "Fengjie Li",
       "Zijie Zhao",
       "Zhirui Ye",
       "Mengjiao Liu",
       "Bo Wang",
       "Hongyu Zhang",
       "Junjie Chen"
     ],
     "year": 2023,
     "venue": "arXiv (cs.SE)",
     "arxiv_id": "2312.15955"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We make all our experimental results and implementations publicly available' and provides a Zenodo link (https://zenodo.org/records/14997209) in reference [40]."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available Defects4J benchmark (v1.2 and v2.0) and states all experimental results are publicly available via the Zenodo archive [40]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Ubuntu 18.04, equipped with 128GB RAM and a processor of Intel(R) Xeon(R) E5-2640' in Section IV-B, but does not provide a requirements.txt, Dockerfile, or detailed library/dependency versions needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a Zenodo archive is referenced, the paper itself does not include step-by-step reproduction instructions, a README with commands, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates for bug counts and precision percentages (e.g., '75 bugs', '54.3% precision') without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims such as 'REPATT significantly outperforms the baseline approaches' and '97.4% and 23.0% more bugs' but uses no statistical significance tests (no p-values, t-tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes in the form of percentage improvements with baseline context, e.g., '15.6%-51.7% higher patch precision', '97.4% and 23.0% more bugs', and '39 more bugs than the best-performing method (i.e., TBar)'. This provides enough context to judge the magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Defects4J v1.2 (395 bugs) and v2.0 (440 bugs) as the benchmark without justifying why this sample size is adequate or discussing power considerations."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. The approach is deterministic (pattern mining is deterministic given the same input), but no discussion of this is provided, and multiple stochastic components (fault localization) exist."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares REPATT against ten state-of-the-art APR approaches: CapGen, SimFix, TransplantFix, TBar, Recoder, SelfAPR, ITER, AlphaRepair, Repilot, and GAMMA (Section IV-A)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include contemporary methods: AlphaRepair (2022), Repilot (2023), GAMMA (2023), SelfAPR (2022), and ITER (2024), which represent the state of the art at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section V-C) presents a thorough ablation study examining the contribution of offline token-level pattern mining, online expression-level code search, the skip-fashion mining, and the S-TAC representation (Table III)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both the number of correctly repaired bugs (recall) and patch precision (ratio of correctly repaired bugs to bugs with plausible patches), as stated in Section IV-B."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B states 'a patch is deemed to be correct iff it is semantically equivalent to the developer patch by manual check. In this process, the first three authors independently conducted the annotation and reached a consensus through discussion.' This constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The approach uses Defects4J v1.2 and v2.0 as separate evaluation sets. The pattern mining is done per-project (on the faulty program itself, not a training set), so there is no train/test contamination concern in the traditional sense. Defects4J v2.0 serves as additional evaluation data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-project breakdowns of bugs repaired for all 18 projects in Defects4J v1.2 and v2.0, rather than just aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A1 discusses incorrect patches: 'When analyzing the incorrect patches generated by REPATT, we found that most of them resulted from the identification of non-representative patterns, which led to incorrect modification locations or inappropriate fixes.' The ablation in RQ3 also shows where components fail (e.g., No S-TAC producing 0 correct patches)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper honestly reports that REPATT 'did not outperform all baselines considering all comparison aspects' (Section V-A1) and repairs fewer bugs than several baselines (75 vs 121 for GAMMA). The No S-TAC ablation shows 0 correct patches, a clear negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about 9 unique bugs vs LLM-based methods, 19 unique bugs vs traditional methods, 83.8% patch precision, and 124 bugs with COMBINE are all supported by results in Tables I, II, and Figure 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (RQ3): removing skip-fashion mining causes 18 fewer bugs repaired, removing S-TAC causes 0 correct patches. These are controlled single-variable manipulations that adequately support the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its claims to the Defects4J benchmark and explicitly discusses generalization limitations in the threats to validity: 'the performance of REPATT may not be generalized to other datasets' (Section VI). The title and claims are scoped to 'redundancy-based automated program repair' rather than making sweeping general claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats-to-validity section (Section VI) discusses methodological threats (manual verification, benchmark selection, reuse of baseline results) but does not discuss specific alternative explanations for the observed results, such as whether improvements might be due to the specific characteristics of Defects4J bugs rather than the approach's general effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "REPATT is not an LLM-based approach and does not use pre-trained language models. It is a traditional pattern-mining-based APR technique. The LLM-based baselines are compared using their published results."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. REPATT is a traditional program analysis approach based on pattern mining, not an LLM-based method."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: MAX_LEN and MAX_SKIP for pattern mining, MIN_SUPPORT threshold (default 3, with sensitivity analysis in RQ4), maximum patches generated (200 token-level, 1000 expression-level), and the 5-hour time budget for repair (Section IV-B, V-D)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. REPATT is a traditional automated program repair tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section II-A documents the data preparation: filtering bugs whose patches involve more than one Java file, resulting in 564 bugs from Defects4J and 609 bugs from Bugs.jar across 24 projects. The token extraction process is described in Section III-A."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Discussion) contains a 'Limitation' subsection and both 'Internal threats to validity' and 'External threats to validity' subsections with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: (1) manual verification of patches by three authors with consensus discussion, (2) reuse of baseline results from published papers without reproduction in a unified environment, (3) evaluation limited to Defects4J v1.2 and v2.0."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VI states specific scope boundaries: 'For complex bugs that require multi-line changes, the reference code may not exist, where our approach will be less effective' and 'the performance of REPATT may not be generalized to other datasets.' The paper also explicitly notes the limitation to Java programs and single-file bugs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We have also made all generated patches publicly available for further inspection and validation' via the Zenodo archive [40], and uses the publicly available Defects4J benchmark."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section II-A describes the data collection: using Defects4J and Bugs.jar benchmarks, filtering for single-file bugs, resulting in 564 and 609 bugs respectively. Section IV-A describes the baseline selection and result collection process."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard public benchmarks (Defects4J)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: faulty program input -> token extraction and embedding -> offline pattern mining -> online code search -> S-TAC transformation -> patch generation -> ranking -> test validation -> manual correctness check (Sections III-IV). Filtering criteria for the preliminary study are also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section discloses funding: 'This work was supported by the National Key Research and Development Program of China (Grant No. 2024YFB4506300), and the National Natural Science Foundation of China (Grant Nos. 62202324, No. 62322208 and No. 62202040).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Tianjin University, University of Pennsylvania, Westlake University, Beijing Jiaotong University, and Chongqing University. None of the authors are affiliated with companies whose products are being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funding comes from the Chinese national science foundations (NSFC and National Key R&D Program), which are independent government funding agencies with no financial stake in the outcome of this specific APR tool evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial interests statement in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "REPATT is not a pre-trained model. It is a traditional pattern-mining approach that mines patterns from the faulty program itself at repair time. It does not have a training data cutoff. The LLM baselines are compared using their published results."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "REPATT does not have a pre-trained model with training data that could overlap with the test benchmark. The pattern mining is done per-project on the faulty program itself, which is the intended use case."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Benchmark contamination is not applicable to REPATT since it does not use pre-trained models. However, the paper does note that LLM-based baselines may face 'data leakage' issues (reference [81] in Section VII)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The manual patch verification by authors is part of the evaluation methodology, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'the construction is actually efficient and on average took about three minutes' for offline pattern mining, and references a 5-hour time budget, but does not report the total wall-clock time per bug, cost per patch, or total computational cost of running REPATT across all bugs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is mentioned (Ubuntu 18.04, 128GB RAM, Intel Xeon E5-2640) but the total computational budget (total CPU hours, total time for all experiments) is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "REPATT repairs 75 bugs on Defects4J with perfect fault localization, complementing existing approaches by repairing 9 unique bugs vs LLM-based/deep learning methods and 19 unique bugs vs traditional methods.",
       "evidence": "Table I shows 75 correctly repaired bugs out of 138 plausible. Figure 5 shows Venn diagrams of unique bug repairs: 19 unique vs traditional (Fig 5a), 9 unique vs LLM/DL methods (Fig 5b), and 5 unique vs all baselines (Fig 5c).",
       "supported": "strong"
     },
     {
       "claim": "REPATT achieves 83.8% patch precision under automated fault localization (SBFL), significantly outperforming baselines by 15.6%-51.7%.",
       "evidence": "Table II shows REPATT at 31/37 correct/plausible = 83.8% precision. The next best is ITER at 68.2%. Other baselines range from 32.1% to 50.0%. The improvement range of 15.6%-51.7% is verified against these numbers.",
       "supported": "strong"
     },
     {
       "claim": "The combined method (COMBINE) repairs 124 bugs at Top-1, 39 more than the best-performing individual method (TBar).",
       "evidence": "Table I shows COMBINE at 124/280 correct/plausible, while TBar is at 85/180. 124-85=39 more bugs. This is confirmed in Section V-B.",
       "supported": "strong"
     },
     {
       "claim": "89.3% of reusable code elements contain fewer than three tokens.",
       "evidence": "Preliminary study in Section II-A with Figure 1 showing token length distributions across 564 bugs from Defects4J and 609 bugs from Bugs.jar.",
       "supported": "moderate"
     },
     {
       "claim": "Both the skip-fashion pattern mining and S-TAC representation are effective components.",
       "evidence": "Table III (Section V-C) shows removing skip reduces correct fixes from 42 to 24 and precision from 65.6% to 44.4%. Removing S-TAC results in 0 correct fixes and 22 incorrect ones.",
       "supported": "strong"
     },
     {
       "claim": "REPATT 'significantly outperforms' baselines under automated fault localization.",
       "evidence": "Table II shows higher precision (83.8%) but fewer correct patches (31 vs 70 for Recoder). The claim of 'significantly outperforms' is scoped to precision but the word 'significantly' is used without statistical testing.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "REPATT, a redundancy-based automated program repair technique using two-level pattern mining (token-level and expression-level), repairs 75 bugs on Defects4J with 54.3% precision under perfect fault localization, complementing existing APR approaches by fixing 5 unique bugs that no baseline can repair. Under realistic automated fault localization, REPATT achieves 83.8% patch precision, substantially higher than all baselines (15.6-51.7% improvement), though at the cost of repairing fewer total bugs. A combined ranking strategy (COMBINE) that merges REPATT with traditional APR tools repairs 124 bugs at Top-1, improving the best individual method by 45.9%.",
   "red_flags": [
     {
       "flag": "Reuse of baseline results without reproduction",
       "detail": "The paper reuses published baseline results from their respective papers rather than reproducing them in a unified environment. This means differences in hardware, configuration, time budgets, or evaluation methodology could affect comparisons. The authors acknowledge this as a threat to validity."
     },
     {
       "flag": "Use of 'significantly' without statistical tests",
       "detail": "The paper uses the term 'significantly outperforms' (Section V-A3) when describing REPATT's precision advantage, but no statistical significance tests are performed. The comparison is based on single-run point estimates."
     },
     {
       "flag": "Recall-precision tradeoff obscured in claims",
       "detail": "REPATT repairs substantially fewer bugs than LLM-based methods (75 vs 121 for GAMMA) but achieves higher precision. The abstract and claims emphasize complementarity and precision without equally emphasizing the recall gap, which could give a misleading impression of overall effectiveness."
     }
   ],
   "cited_papers": [
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair is a key baseline; represents LLM-based zero-shot program repair relevant to AI-assisted code generation."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Y. Wei", "C. S. Xia", "L. Zhang"],
       "year": 2023,
       "relevance": "Repilot is a baseline that integrates LLMs with completion engines for program repair, directly relevant to LLM-based software engineering."
     },
     {
       "title": "Gamma: Revisiting template-based automated program repair via mask prediction",
       "authors": ["Q. Zhang", "C. Fang", "T. Zhang", "B. Yu", "W. Sun", "Z. Chen"],
       "year": 2023,
       "relevance": "GAMMA uses CodeBERT and UniXcoder for template-based APR, relevant to evaluating LLM-based code repair methods."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "relevance": "Directly evaluates the impact of code language models on program repair, relevant to AI-assisted programming evaluation."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Surveys automated program repair using large pre-trained language models, directly relevant to LLM capability evaluation."
     },
     {
       "title": "Breaking the silence: the threats of using LLMs in software engineering",
       "authors": ["J. Sallou", "T. Durieux", "A. Panichella"],
       "year": 2024,
       "relevance": "Discusses data leakage and other threats when using LLMs in software engineering research, relevant to methodology quality."
     },
     {
       "title": "A systematic literature review on large language models for automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Xie", "Y. Ma", "W. Sun", "Y. Yang", "Z. Chen"],
       "year": 2024,
       "doi": "arXiv:2405.01466",
       "relevance": "Systematic review of LLMs for APR, relevant to survey methodology and LLM capability assessment."
     },
     {
       "title": "Evaluating the generalizability of LLMs in automated program repair",
       "authors": ["F. Li", "J. Jiang", "J. Sun", "H. Zhang"],
       "year": 2025,
       "relevance": "Evaluates LLM generalizability for program repair, directly relevant to AI capability evaluation methodology."
     },
     {
       "title": "Hybrid automated program repair by combining large language models and program analysis",
       "authors": ["F. Li", "J. Jiang", "J. Sun", "H. Zhang"],
       "year": 2024,
       "relevance": "Combines LLMs with traditional program analysis for repair, relevant to hybrid AI-assisted programming approaches."
     },
     {
       "title": "The plastic surgery hypothesis in the era of large language models",
       "authors": ["C. S. Xia", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "relevance": "Revisits the plastic surgery hypothesis (code redundancy for repair) in the context of LLMs, directly relevant to evaluating LLM-based code generation."
     }
   ]
 }