ai-research-survey

scan.json (25271B)

---

 {
   "paper": {
     "title": "A Survey of Learning-based Automated Program Repair",
     "authors": [
       "Quanjun Zhang",
       "Chunrong Fang",
       "Yuxiang Ma",
       "Weisong Sun",
       "Zhenyu Chen"
     ],
     "year": 2023,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2301.03270",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic review of 112 learning-based APR papers (2016–2022) finds the field has grown rapidly since 2020, with Java dominating (44%) as the target language. The survey categorizes techniques by code representation (sequence, tree, graph), documents 53 datasets and two metric families (execution-based and match-based), and identifies pre-trained models as the dominant recent trend. A critical open science audit reveals many papers fail to release source code, datasets, or trained models, hindering reproducibility.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a public GitHub repository: https://github.com/iSEngLab/AwesomeLearningAPR, mentioned in the abstract and Section 1 ('Our artifacts are publicly available at the repository')."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'All artifacts of this study are available' at the GitHub repository, which serves as the curated dataset of 112 collected papers and their classifications."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. As a survey paper with no computational analysis scripts, no environment setup is documented."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 provides a detailed search methodology: explicit search strings for two keyword groups, three databases (Google Scholar, ACM, IEEE), filtering criteria (before 2016, less than 7 pages, deduplication), and a snowballing procedure, shown in Figure 1."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or produce statistical results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that makes no comparative statistical claims requiring significance testing."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental results requiring effect size reporting."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper; the number of collected papers (112) is determined by the search and filtering process, not by statistical power considerations."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental runs requiring variance reporting."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against prior surveys by Gazzola et al. [53] (up to January 2017) and Monperrus et al. [135], explicitly distinguishing its focus on learning-based APR and its more recent coverage (Section 1, 'Comparison with Existing Surveys')."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The compared prior surveys (Gazzola et al. 2019, Monperrus 2018) are the most relevant existing APR surveys, and the paper clearly explains why a new survey focusing on learning-based techniques is needed."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run experiments or report evaluation metrics for its own system."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no system outputs to evaluate."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no experimental evaluation requiring train/test splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The survey provides extensive breakdowns: papers by year (Figure 2), by programming language (Figure 3), by bug type (semantic/syntax/vulnerability), by code representation (sequence/tree/graph), by model architecture, by dataset, and by tool availability (Tables 1-7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.3 discusses the open science problem in detail, identifying papers that fail to release artifacts. Section 8 discusses challenges and limitations of existing learning-based APR techniques across 10 guideline areas."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports concerning findings: the open science problem (Section 7.3), the overfitting issue (Section 4.7), that match-based metrics may be improper for APR (Section 6.2), and that many training datasets contain noise (Section 8, I&G3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to survey learning-based APR, detail crucial components, discuss datasets/metrics, summarize empirical studies, discuss repair domains and industrial deployment, and provide guidelines. All these are addressed in Sections 2–8 of the paper."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a survey that reports observations and trends. It does not make causal claims about interventions; interpretive statements about why trends occur ('One reason behind this phenomenon is...') are speculative observations, not tested causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states its scope: papers collected through November 2022, focused on learning-based APR (not traditional APR), with the search process described in Section 2. The title clearly bounds the scope to 'Learning-based Automated Program Repair.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its survey findings. For example, the observed growth trend in learning-based APR papers could reflect publication venue expansion or keyword inflation, but no such alternatives are considered. There is no threats-to-validity section for the survey methodology itself."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match the granularity of its measurements. It counts papers, categorizes techniques, and reports trends — it does not claim these proxies represent broader constructs beyond what is directly measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use any ML models."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no models or hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 documents the paper selection pipeline: automated search (342 papers) → filter by year and page count (283) → manual screening for relevance (87) → snowballing (112). The search string, databases, and filtering criteria are explicitly stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section for the survey itself. Section 8 discusses limitations of the surveyed APR techniques (not the survey methodology), and Section 9 is a brief conclusion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed for the survey methodology itself. Potential issues such as search string completeness, manual screening reliability, or temporal bias in paper collection are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states it covers papers through November 2022 (Section 2), focuses on learning-based APR (distinct from traditional APR), and explicitly compares its scope with prior surveys by Gazzola et al. and Monperrus et al. in Section 1."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The artifact repository at https://github.com/iSEngLab/AwesomeLearningAPR provides the collected papers list and tool availability data for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 describes the data collection in detail: search databases (Google Scholar, ACM, IEEE), search string with two keyword groups, time period ('end of November 2022'), and the filtering process with specific criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are academic databases and digital libraries."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Section 2 document the full pipeline: automated search (342 papers) → year and page filtering with deduplication (283) → manual relevance screening (87) → snowballing of missed citations (112). Each stage has counts and criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists funding: National Natural Science Foundation of China (61932012, 62141215, 62372228), CCF-Huawei Populus Grove Fund, and Shenzhen Municipality commission."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from State Key Laboratory for Novel Software Technology, Nanjing University. They are not evaluating their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from NSFC (government grants) and CCF-Huawei academic fund. While Huawei has software engineering interests, the survey's findings do not evaluate Huawei products. The funders have no obvious stake in the survey's conclusions."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate a pre-trained model on any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 explicitly follows Petersen et al. [153] and Kitchenham et al. [82] systematic review methodology. The paper provides a structured search strategy with reproducible queries (exact search string provided), three databases, explicit filtering criteria, and a snowballing phase. Figure 1 shows the workflow diagram."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not assess the methodological quality of the 112 included papers. All papers are treated equally regardless of their experimental rigor, sample sizes, or evaluation quality. Tables summarize techniques and datasets but include no quality scoring."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The survey does not discuss publication bias. There is no consideration of whether the collected papers skew toward positive results, no funnel plots, and no acknowledgment that published APR papers may disproportionately report successful repair techniques."
       }
     }
   },
   "claims": [
     {
       "claim": "112 relevant learning-based APR studies were collected from 2016 to 2022, with rapid growth since 2020 (47 papers in 2022 alone).",
       "evidence": "Section 2 describes the collection methodology; Figure 2 shows the year-by-year distribution: 3 (2016), 4 (2017), 6 (2018), 13 (2019), 13 (2020), 25 (2021), 47 (2022).",
       "supported": "strong"
     },
     {
       "claim": "Java is the most targeted programming language in learning-based APR, accounting for 44% of papers.",
       "evidence": "Figure 3 shows the language distribution: Java 44%, C 20%, Python 18%, JavaScript 13%, C++ 5%.",
       "supported": "strong"
     },
     {
       "claim": "Pre-trained models have significantly influenced program repair, with techniques divided into universal (e.g., CodeT5) and specific (e.g., AlphaRepair) categories.",
       "evidence": "Section 5 provides a comprehensive taxonomy of pre-trained model-based APR techniques with Table 3 listing specific approaches (TFix, CIRCLE, VulRepair, AlphaRepair, SYNSHINE).",
       "supported": "strong"
     },
     {
       "claim": "53 datasets are used across learning-based APR studies, with Defects4J being the most widely adopted.",
       "evidence": "Table 4 lists all 53 datasets with their languages, sizes, and which studies use them. Defects4J appears in the most studies' evaluation columns.",
       "supported": "strong"
     },
     {
       "claim": "Many learning-based APR papers fail to provide adequate open science artifacts (source code, datasets, or trained models).",
       "evidence": "Section 7.3 and Table 7 audit tool availability. Several papers fail to provide source code, datasets, or trained models. Some studies cannot be reproduced due to missing hyperparameters, complex environment settings, or insufficient documentation.",
       "supported": "moderate"
     },
     {
       "claim": "The overfitting issue is more significant in learning-based APR than in traditional APR due to the black-box nature of end-to-end NMT repair.",
       "evidence": "Section 4.7 discusses this: 'The overfitting issue in learning-based APR is more significant and severe' citing [198], with examples of plausible patches that pass test suites but are not semantically correct.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No quality assessment of included studies",
       "detail": "The survey collects 112 papers but applies no quality scoring rubric or risk-of-bias assessment. All papers are treated equally regardless of their methodological quality, sample sizes, or evaluation rigor. This risks laundering weak results alongside strong ones."
     },
     {
       "flag": "No threats to validity for the survey itself",
       "detail": "The paper has no limitations or threats-to-validity section addressing its own methodology. Issues such as search string completeness, potential missed papers despite snowballing, inter-rater reliability of manual screening, or temporal bias are not discussed."
     },
     {
       "flag": "Publication bias not addressed",
       "detail": "The survey does not consider whether the 112 collected papers skew toward positive results. Published APR papers may disproportionately report successful techniques, inflating the apparent progress of the field."
     },
     {
       "flag": "Uncritical repetition of claimed results",
       "detail": "The survey reports claimed bug counts and performance numbers from individual papers (e.g., 'CoCoNut fixes 509 bugs') without verifying these claims or noting that different papers use different evaluation settings, making direct comparisons misleading."
     }
   ],
   "cited_papers": [
     {
       "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "relevance": "First extensive evaluation of 9 large pre-trained language models for automated program repair across 3 languages, directly relevant to LLM capability assessment."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair Via Zero-shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair demonstrates zero-shot APR with CodeBERT, showing pre-trained models can repair without task-specific fine-tuning — key evidence for LLM capability claims."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq Joty", "Steven CH Hoi"],
       "year": 2021,
       "relevance": "Foundational pre-trained model for code tasks including program repair, widely used as backbone in APR systems."
     },
     {
       "title": "CodeBERT: A Pre-trained Model for Programming and Natural Languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "relevance": "Influential pre-trained model used across multiple APR techniques for code understanding and generation."
     },
     {
       "title": "VulRepair: A T5-based Automated Software Vulnerability Repair",
       "authors": ["Michael Fu", "Chakkrit Tantithamthavorn", "Trung Le", "Van Nguyen", "Phung Dinh"],
       "year": 2022,
       "relevance": "Applies pre-trained CodeT5 to security vulnerability repair, relevant to AI safety and automated code security."
     },
     {
       "title": "CIRCLE: Continual Repair across Programming Languages",
       "authors": ["Wei Yuan", "Quanjun Zhang", "Tieke He", "Chunrong Fang"],
       "year": 2022,
       "relevance": "Demonstrates continual learning for multi-language program repair with T5, relevant to LLM generalization across programming languages."
     },
     {
       "title": "CURE: Code-aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "Combines pre-trained programming language model with code-aware beam search for APR, influential in NMT-based repair."
     },
     {
       "title": "Recoder: A Syntax-guided Edit Decoder for Neural Program Repair",
       "authors": ["Qihao Zhu", "Zeyu Sun", "Yuan-an Xiao"],
       "year": 2021,
       "relevance": "First learning-based APR technique to outperform traditional techniques on Defects4J, demonstrating the maturation of neural repair approaches."
     },
     {
       "title": "Defects4J: A Database of Existing Faults to Enable Controlled Testing Studies for Java Programs",
       "authors": ["René Just", "Darioush Jalali", "Michael D Ernst"],
       "year": 2014,
       "relevance": "The most widely used benchmark in APR research; its characteristics shape how the entire field evaluates repair techniques."
     },
     {
       "title": "Break-It-Fix-It: Unsupervised Learning for Program Repair",
       "authors": ["Michihiro Yasunaga", "Percy Liang"],
       "year": 2021,
       "relevance": "Demonstrates unsupervised learning for program repair using a fixer-breaker loop, relevant to self-supervised AI code generation."
     },
     {
       "title": "Self-supervised Bug Detection and Repair",
       "authors": ["Miltiadis Allamanis", "Henry Jackson-Flux", "Marc Brockschmidt"],
       "year": 2021,
       "relevance": "BUGLAB from Microsoft: self-supervised bug detection and repair using detector-selector learning, demonstrating industrial interest in automated code repair."
     },
     {
       "title": "An Empirical Study on Learning Bug-fixing Patches in the Wild Via Neural Machine Translation",
       "authors": ["Michele Tufano", "Cody Watson", "Gabriele Bavota", "Massimiliano Di Penta", "Martin White", "Denys Poshyvanyk"],
       "year": 2019,
       "relevance": "First systematic empirical study on NMT for bug-fixing, establishing foundational datasets (BFP) used across the field."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a useful taxonomy and reading list for researchers entering the learning-based APR field, with a curated GitHub repository, but is not an immediately usable tool."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected narrative of growing interest in DL for APR without challenging any conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the survey covers vulnerability repair as one application domain but does not raise novel risk concerns."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper is a straightforward literature survey."
     },
     "demo_ability": {
       "score": 1,
       "justification": "GitHub repository with a curated paper list is available but there is no interactive tool or demo to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Published in TOSEM (reputable SE venue) by Nanjing University researchers, but not from a high-profile AI lab."
     }
   }
 }