ai-research-survey

scan.json (28136B)

---

 {
   "paper": {
     "title": "A Survey on Automated Program Repair Techniques",
     "authors": [
       "Kai Huang",
       "Zhengzi Xu",
       "Su Yang",
       "Hongyu Sun",
       "Xuejun Li",
       "Zheng Yan",
       "Yuqing Zhang"
     ],
     "year": 2023,
     "venue": "ACM Computing Surveys",
     "arxiv_id": "2303.18184",
     "doi": "10.48550/arXiv.2303.18184"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis", "qualitative"],
   "key_findings": "This survey classifies APR techniques into four categories (search-based, constraint-based, template-based, learning-based) and reviews 140 representative works from 2005-2022. The paper finds that learning-based APR tools now outperform traditional techniques on the Defects4J benchmark, and that LLMs are transforming the field through both fine-tuning and prompt learning paradigms. The survey reveals significant methodological problems in APR research including unfair experimental comparisons due to inconsistent training datasets, dataset overlap (data leakage) between training and test sets, and dataset quality issues (e.g., 20.71% invalid samples in CoCoNut). Key challenges remain in multi-hunk repair, patch overfitting, and industrial deployment.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No analysis code or scripts for the survey are released. A GitHub URL is provided for supplementary dataset overlap data (https://github.com/huangkNIPC/APR-Survey/tree/main/dataset_overlap), but no code for reproducing the survey's analysis."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides some dataset overlap data on GitHub, but does not release the full structured survey dataset (paper list, evaluation criteria results, or analysis data) in a downloadable format. Tables in the paper constitute the primary data presentation."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications provided for the preliminary experiments (Tables 5, 7, 9). No requirements.txt, Dockerfile, or environment details are given."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for either the survey methodology or the preliminary experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is primarily a survey paper. The preliminary experiments (Tables 5, 7) report point estimates but are supplementary, not the main contribution."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "As a survey paper, the main contribution is taxonomic and qualitative. The preliminary experiments in Findings 7 and 14 make comparative claims without statistical tests, but these are supplementary observations."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; statistical effect sizes are not applicable to the primary taxonomic analysis."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. The selection of 140 papers is explained by page limitations and manual selection criteria, but formal sample size justification is not applicable."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. The preliminary experiments report single-run results without variance, but these are supplementary."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The survey explicitly compares against prior surveys by Gazzola et al. [30], Le Goues et al. [35], and Monperrus [96, 97], and positions its contributions relative to these prior works in Section 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared prior surveys include Monperrus's living review (2022) [97] and Gazzola et al. (2019) [30], which are contemporary. The paper explicitly states it fills gaps left by these recent surveys."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no system is being evaluated with metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; human evaluation of system outputs is not applicable."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no train/test split is applicable."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides detailed per-category breakdowns: four main APR categories (search-based, constraint-based, template-based, learning-based) with subcategories. Table 2 compares tools across 7 metrics, Tables 10-13 provide per-category summaries, and Table 3 summarizes pros and cons by approach."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure cases and limitations of each APR approach class in Table 3 (pros/cons), Section 5.5 (findings revealing problems), and Section 7.1 (existing challenges including patch overfitting, repair quality, efficiency problems)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative findings are reported: Code Abs does not significantly improve LLM repair (Table 5, Finding 7), dataset overlap problems exist (Table 9, Finding 15), 20.71% invalid samples in CoCoNut dataset (Finding 15), and unfair experimental comparisons across APR works (Finding 11, 15)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a systematic taxonomy, uniform evaluation criteria, and discussion of future directions — all of which are delivered in the paper body. The taxonomy is in Section 5, criteria in Section 4 (Table 1), and challenges/directions in Section 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Finding 7 claims 'Code Abs does not work well on LLMs' based on a preliminary experiment (Table 5) without statistical tests or controlled design. Finding 5 claims 'more accurate FL might further alleviate the patch overfitting problem' based on observational comparison (Table 4) without controlling for confounds. These causal claims are not adequately justified."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper bounds its scope: papers up to December 2022, 140 selected works, focus on APR for software bugs/errors/vulnerabilities. Section 2 explicitly notes page limitations and possible omissions: 'it is impossible to cover all typical works in our survey.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Finding 7 provides two explanations for why Code Abs doesn't help LLMs: (1) code abstraction loses semantic information from method/variable names, and (2) LLMs are pre-trained on raw source code. Finding 5 notes multiple factors affecting patch overfitting including FL accuracy, patch generation, and validation."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between different measurement approaches: correct patches vs. plausible patches, repair accuracy (exact match) vs. test suite validation. Finding 13 explicitly discusses limitations of exact match as a validation strategy. The paper's evaluation criteria (Section 4) carefully distinguish between different quality and effectiveness measures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. The preliminary experiments reference LLMs (CodeBERT, GraphCodeBERT, PLBART, CodeT5) by name but defer to prior work [28, 161] for implementation details. Model versioning is not the focus."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting in its own methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper. The preliminary experiments reference prior work for implementation details without reporting hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the paper selection pipeline: collection from APR community (405 papers), Monperrus living review (401 papers), and own search on Google Scholar/DBLP/arXiv using keywords 'Automated/Automatic Program Repair' for 2022. Filtering excluded short papers (≤6 pages), then manual review selected 140 works. Fig. 1 shows the workflow."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section for the survey methodology itself. Section 7 discusses challenges and directions for APR research generally, not limitations of the survey's own methodology (selection bias, coverage gaps, etc.)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to the validity of the survey itself are discussed. The paper does not address potential selection bias in their manual paper filtering, coverage limitations, or how their subjective 'typical works' criterion might skew the findings."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states temporal scope (papers up to December 2022), paper count (140 selected works), and focus areas (data-driven APR techniques). Section 2 notes: 'Due to page limitations, we cannot review all APR works in detail' and acknowledges 'possible omissions from the manual review.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The full list of surveyed papers, evaluation criteria assessments, and analysis data are not available as a downloadable dataset. Only partial data on dataset overlap is provided via GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes data collection: three sources (APR community website, Monperrus living review, own search), time period (up to December 2022), search keywords, and filtering steps. Fig. 1 provides a workflow diagram."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Section 2 describes the pipeline stages (collection → dedup → filter short papers → manual review → 140 works) and gives input counts (405 + 401 papers), intermediate counts are missing (how many after dedup? how many after short paper filter?). The manual selection criteria are vague: 'typical works' that 'represent the characteristics of certain types of techniques.'"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed: 'National Natural Science Foundation of China under Grant 62072351' and 'Academy of Finland under Grant 345072 and Grant 350464.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Xidian University, Nanyang Technological University, University of Chinese Academy of Sciences, and Aalto University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government/academic research agencies (Chinese NSF, Academy of Finland) with no commercial stake in APR tool outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model's capability on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate a pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no system with inference cost to report."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper; no significant compute budget applies."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "Section 2 and Fig. 1 describe a paper selection workflow with defined sources and keywords, but the core filtering step relies on subjective manual selection of 'typical works' without reproducible inclusion/exclusion criteria. The protocol is semi-structured but not systematic enough for PRISMA compliance — no registered protocol, and the manual selection criteria are undefined beyond 'represent the characteristics of certain types of techniques.'"
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey proposes evaluation criteria for APR tools (Table 1, Section 4) but does not systematically apply quality scoring to the 140 reviewed papers. All papers are treated with equal weight regardless of their methodological rigor, and no risk-of-bias assessment is performed on included studies."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The survey does not consider whether its sources skew toward positive results, does not include funnel plots, and does not acknowledge that published APR papers may over-represent successful repair approaches."
       }
     }
   },
   "claims": [
     {
       "claim": "Learning-based APR tools outperform traditional APR tools on the Defects4J benchmark",
       "evidence": "Finding 6, Table 4 (Section 5.5.2): AlphaRepair achieves 74/109 correct/plausible patches with PFL on Defects4J V1.2.0, compared to TBar (68/95) and jGenProg (6/16). Multiple learning-based tools exceed traditional ones.",
       "supported": "moderate"
     },
     {
       "claim": "Code abstraction (Code Abs) does not significantly improve the repair performance of most LLMs",
       "evidence": "Finding 7, Table 5: Preliminary experiments fine-tuning CodeBERT, GraphCodeBERT, PLBART, and CodeT5 on BFP dataset with and without Code Abs. Most LLMs show similar or lower performance with Code Abs. However, no statistical tests are reported.",
       "supported": "weak"
     },
     {
       "claim": "Using BLEU as model evaluation metric can improve repair accuracy to 55% vs. 44% with loss on VulRepair",
       "evidence": "Finding 14, Table 7: Preliminary experiment fine-tuning CodeT5 on VulRepair dataset. Best BLEU model achieves 55% accuracy vs. 44% for VulRepair's original loss-based selection.",
       "supported": "weak"
     },
     {
       "claim": "Multiple large-scale training datasets have data overlap with the Defects4J test benchmark",
       "evidence": "Finding 15, Table 9: Exact matching detected overlaps — MegaDiff overlaps with 99 bugs in Defects4J, CodRep with 61, CodeSearchNet with 48. Data and method provided via GitHub link.",
       "supported": "moderate"
     },
     {
       "claim": "The CoCoNut dataset contains 20.71% invalid or incorrect samples",
       "evidence": "Finding 15: '671,497 (671,497/3,241,966=20.71%) invalid or incorrect samples in CoCoNut, and the bug codes and fix codes in these samples were identical.'",
       "supported": "moderate"
     },
     {
       "claim": "More accurate fault localization may further alleviate the patch overfitting problem",
       "evidence": "Finding 5, Table 4: APR tools using perfect fault localization (PFL) generally show higher percentages of correct patches vs. those using off-the-shelf FL tools. E.g., AlphaRepair: 67.89% with PFL vs. 55.56% without.",
       "supported": "moderate"
     },
     {
       "claim": "Multi-hunk bug repair remains a long-term challenge with three main approaches showing distinct limitations",
       "evidence": "Finding 18, Fig. 7: Reviews iterative, holistic, and synchronous repair approaches. Iterative repair creates exponential search spaces; holistic repair treats bug code and context uniformly; synchronous repair (DEAR) requires AST conversion limiting applicability.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Subjective manual paper selection",
       "detail": "The survey's core filtering step uses manual selection of 'typical works' without reproducible inclusion/exclusion criteria. This introduces selection bias and makes the survey non-reproducible. The authors acknowledge 'possible omissions from the manual review' but do not discuss how this affects their findings."
     },
     {
       "flag": "No quality assessment of source papers",
       "detail": "The survey does not assess the methodological quality of the 140 reviewed papers. All works are treated with equal weight regardless of their rigor, potentially laundering the signal-to-noise ratio of its sources. The proposed evaluation criteria (Table 1) are not applied as a quality assessment."
     },
     {
       "flag": "Preliminary experiments lack statistical rigor",
       "detail": "Tables 5, 7, and 9 present quantitative findings from preliminary experiments without error bars, significance tests, multiple runs, or variance reporting. The claim that 'Code Abs does not significantly improve repair performance' (Finding 7) is made without any statistical significance test."
     },
     {
       "flag": "No survey limitations discussed",
       "detail": "The paper lacks a limitations section for its own survey methodology. Section 7 discusses challenges in the APR field, but does not address the survey's own potential biases, coverage gaps, or methodological limitations."
     },
     {
       "flag": "Incomplete data pipeline documentation",
       "detail": "The paper collection pipeline (Section 2) omits intermediate counts: starting from 405+401+additional papers, the final count is 140, but the number after deduplication and after short-paper filtering are not reported, making it impossible to assess the filtering funnel."
     }
   ],
   "cited_papers": [
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Proposes AlphaRepair, demonstrating LLM zero-shot learning for program repair using CodeBERT, outperforming all prior APR tools on Defects4J."
     },
     {
       "title": "VulRepair: a T5-based automated software vulnerability repair",
       "authors": ["Michael Fu", "Chakkrit Tantithamthavorn", "Trung Le", "Van Nguyen", "Dinh Phung"],
       "year": 2022,
       "relevance": "Fine-tunes CodeT5 for automated vulnerability repair, demonstrating LLM fine-tuning as an effective approach for security bug fixing."
     },
     {
       "title": "CIRCLE: continual repair across programming languages",
       "authors": ["Wei Yuan", "Quanjun Zhang", "Tieke He", "Chunrong Fang"],
       "year": 2022,
       "relevance": "Uses continual learning to achieve cross-language program repair in a single model, addressing multi-language repair scalability."
     },
     {
       "title": "Repair is nearly generation: Multilingual program repair with llms",
       "authors": ["Harshit Joshi", "José Cambronero", "Sumit Gulwani", "Vu Le"],
       "year": 2022,
       "arxiv_id": "2208.11640",
       "relevance": "Demonstrates LLM-based multilingual program repair using Codex with few-shot learning across 6 programming languages."
     },
     {
       "title": "Repairing Bugs in Python Assignments Using Large Language Models",
       "authors": ["Jialu Zhang", "José Cambronero", "Sumit Gulwani", "Vu Le"],
       "year": 2022,
       "arxiv_id": "2209.14876",
       "relevance": "Proposes MMAPR system using Codex with multi-modal prompts for automated program error repair, demonstrating LLM prompt engineering for repair."
     },
     {
       "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2210.14179",
       "relevance": "Large-scale empirical study exploring LLM capabilities for practical program repair in zero/few-shot settings."
     },
     {
       "title": "Examining Zero-Shot Vulnerability Repair with Large Language Models",
       "authors": ["Hammond Pearce", "Benjamin Tan", "Baleegh Ahmad", "Ramesh Karri", "Brendan Dolan-Gavitt"],
       "year": 2022,
       "relevance": "Empirical study of LLM zero-shot capability for security vulnerability repair, relevant to understanding LLM limitations in code security."
     },
     {
       "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "Introduces pre-trained programming language model with code-aware beam search for program repair, pioneering the pre-training+fine-tuning paradigm in APR."
     },
     {
       "title": "SapFix: automated end-to-end repair at scale",
       "authors": ["Alexandru Marginean", "Johannes Bader", "Satish Chandra"],
       "year": 2019,
       "relevance": "First automated end-to-end program repair deployed at scale in Meta's industrial CI/CD environment."
     },
     {
       "title": "Getafix: learning to fix bugs automatically",
       "authors": ["Johannes Bader", "Andrew Scott", "Michael Pradel", "Satish Chandra"],
       "year": 2019,
       "relevance": "Facebook's industrial APR tool using static analysis instead of test cases, demonstrating practical deployment of automated repair."
     },
     {
       "title": "Neural Transfer Learning for Repairing Security Vulnerabilities in C Code",
       "authors": ["Zimin Chen", "Steve Kommrusch", "Martin Monperrus"],
       "year": 2022,
       "relevance": "Proposes VRepair using transfer learning from bug repair to vulnerability repair, demonstrating cross-domain knowledge transfer for code fixing."
     },
     {
       "title": "TFix: Learning to Fix Coding Errors with a Text-to-Text Transformer",
       "authors": ["Berkay Berabi", "Jingxuan He", "Veselin Raychev", "Martin T. Vechev"],
       "year": 2021,
       "relevance": "Uses pre-trained T5 model fine-tuned for JavaScript error repair with multi-task learning, demonstrating NL model transfer to PL repair."
     },
     {
       "title": "A syntax-guided edit decoder for neural program repair",
       "authors": ["Qihao Zhu", "Zeyu Sun", "Yuan-an Xiao", "Wenjie Zhang"],
       "year": 2021,
       "relevance": "Proposes Recoder, the first DL-based APR approach outperforming traditional techniques on Defects4J, using syntax-guided decoding."
     },
     {
       "title": "An extensive study on pre-trained models for program understanding and generation",
       "authors": ["Zhengran Zeng", "Hanzhuo Tan", "Haotian Zhang", "Jing Li"],
       "year": 2022,
       "relevance": "Large-scale study comparing pre-trained code models (CodeBERT, GraphCodeBERT, PLBART, CodeT5) on program understanding and generation tasks."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Survey provides a useful taxonomy and comparison of APR tools, but is not directly usable as a tool or technique by practitioners."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Reveals dataset quality and overlap problems in APR research that are somewhat surprising, but largely confirms known trends about LLM dominance."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; focused on software repair methodology."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Points out unfair experimental comparisons, dataset overlap, and 20.71% invalid data in CoCoNut, but frames these diplomatically as findings rather than accusations."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released; purely a literature review paper."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Xidian University and NTU; no famous lab or product brand recognition."
     }
   }
 }