ai-research-survey

scan.json (25111B)

---

 {
   "paper": {
     "title": "A Survey of LLM-based Software Repair: Taxonomies, Design Paradigms, and Applications",
     "authors": [
       "Boyang Yang",
       "Zijian Cai",
       "Fengling Liu",
       "Bach Le",
       "Lingming Zhang",
       "Tegawendé F. Bissyandé",
       "Yang Liu",
       "Haoye Tian"
     ],
     "year": 2025,
     "venue": "ACM Transactions on Software Engineering and Methodology",
     "arxiv_id": "2506.23749",
     "doi": "10.1145/nnnnnnn.nnnnnnn"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL (https://github.com/GLEAM-Lab/ProgramRepair) multiple times (Section 1, Section 2.2, Section 2.4) containing their scripted survey pipeline, screening scripts, per-stage record lists, and analysis notebooks."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that their replication package includes per-stage record lists, extracted metadata, the representation papers list, and analysis notebooks. The artifacts at the GitHub repository are described as 'publicly available' (Section 1)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications (requirements.txt, library versions, etc.) are mentioned in the paper for running their survey pipeline scripts."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the paper mentions scripts and notebooks are available, no step-by-step reproduction instructions are provided in the paper itself. The paper states the artifacts are 'publicly available' but does not describe how to execute the pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not run its own experiments. It reports descriptive counts and published results from other papers, not its own statistical estimates."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "As a systematic survey, this paper does not make comparative claims based on its own experiments. It summarizes published results from other papers."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run its own experiments; it summarizes effect sizes reported by the surveyed papers."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "As a survey, the paper does not run experiments with sample sizes. The corpus size (62 papers) is the result of a systematic screening process rather than a statistical sample."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; this is a survey paper that reports published results from other studies."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares its taxonomy and coverage against nine prior surveys in Table 8 (Section 9, Related Work), identifying specific gaps each prior survey has and how this survey addresses them."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The prior surveys compared in Table 8 range from 2022 to 2025, including recent works by Zhang et al. (2024), Haque et al. (2025), He et al. (2025), and Zhou et al. (2025). These are contemporary and relevant comparisons."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "As a survey paper, there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run its own evaluation experiments. It reports metrics from surveyed papers (pass@k, accuracy, F1, etc.)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper classifying and comparing existing systems. Human evaluation of system outputs is not relevant to the paper's claims, which are about taxonomy organization and coverage gaps."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; this is a survey."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides per-paradigm breakdowns (Tables 3-9), per-year adoption counts (Table 4), per-benchmark trend snapshots (Tables 5-7), and per-sub-paradigm detailed system information (Table 9). Figure 4 shows defect scope distribution and Figure 5 shows venue distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses limitations and failure modes of each paradigm throughout Sections 4-7. Section 10 (Open Challenges) systematically addresses bottlenecks including evaluation reliability issues, workflow robustness problems, retrieval failures, and security concerns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings: prompt sensitivity causing repair failures (Section 5.1), diminishing returns with verbose context (Section 5.1), benchmark contamination inflating scores (Section 10), weak test suites hiding semantic faults (Section 10), and high coordination costs offsetting accuracy gains in agentic systems (Section 7.3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims a unified taxonomy of 62 systems into four paradigms with two augmentation layers, consolidation of evaluation practice on benchmarks, and public artifact release. All are supported: Table 3 maps all 62 systems, Tables 5-7 consolidate benchmark results, and the GitHub URL is provided."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make causal claims. It describes taxonomic organization, adoption trends, and benchmark results from other papers. Language like 'adoption shifts' describes observed patterns, not causal effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it treats the 62 papers as 'a representative sample of current LLM-based software repair research at these venues rather than a complete survey of all existing systems' (Section 2.4). External validity threats are discussed in Section 11.2."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Answer to RQ3) explicitly discusses how protocol differences, not just model quality, shape reported repair rates. The paper notes that improvements may arise from 'better models, from more generous pass@k, or from stronger evaluation setups' (Section 8). Section 11 discusses threats to validity including internal, external, and construct validity."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not use any LLM models in its own methodology. It reports model versions used by the surveyed papers."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in its own methodology; it surveys papers that use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No LLM experiments are conducted. The paper's methodology is a manual systematic review process."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey's own methodology."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents its filtering pipeline in detail: 1,581 initial hits → 994 after repair-related filter → 726 after LLM-related filter → 449 after benchmark-related filter → 462 after snowballing → 62 representative works (Table 2). Filtering criteria at each stage are described in Sections 2.2-2.4, with inclusion/exclusion criteria explicitly stated in Section 2.3. Keyword families are listed in Table 1."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 11 ('Threats to Validity') presents a dedicated three-part discussion covering internal, external, and construct validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: representation focuses on tier-1 venues (excluding proprietary-data-only tools), headline scores reported as published without normalization of pass@k or budgets, no joint reevaluation performed, and coding scheme ambiguity is mitigated by inter-rater discussion (Sections 11.1-11.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states its scope boundaries: year filter >=2022, tier-1 SE/AI/Security venues, papers with reproducible benchmark evaluation, 62 representative rather than exhaustive sample. Section 2.3 lists detailed exclusion criteria. Section 11.2 states 'tools evaluated only on proprietary data fall outside our scope.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states that 'Per-stage record lists, the scripts for fetching and filtering papers from multiple sources, and analysis notebooks that regenerate our tables and figures are provided in our replication package' (Section 2.4)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes the data collection procedure in detail: structured queries on ACM DL, IEEE Xplore, DBLP, and arXiv; query templates combining core repair terms, LLM names, benchmark names, and tool names (Table 1); year filter >=2022; last query run on 31 October 2025; backward citation snowballing."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The 'sample' is a corpus of research papers collected via database queries and snowballing."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 shows the full pipeline workflow. Table 2 reports exact record counts at each filtering stage (1,581 → 994 → 726 → 449 → 462 → 62). Section 2.4 describes each stage including automatic filters, snowballing, and manual screening with inter-rater agreement via Cohen's kappa."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. There is no mention of funding sources."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed prominently on the first page. Several authors (Boyang Yang, Haoye Tian) are authors of multiple surveyed systems (CREF, KGCompass, MORepair, LANTERN), and these affiliations are visible through the reference list."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed at all. Without knowing the funding source, independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark. (The paper does discuss contamination as an issue in the surveyed works, but this checklist item is about the paper's own methodology.)"
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper that does not evaluate any pre-trained model on a benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. It does not propose or run its own method with inference costs. It reports costs from surveyed papers (e.g., ChatRepair at ~$0.42 per bug)."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper. Its own methodology involves literature search and manual screening, not compute-intensive model runs."
       }
     }
   },
   "claims": [
     {
       "claim": "Adoption of LLM-based software repair paradigms has shifted from single-turn prompting toward procedural and agentic workflows between 2022 and October 2025.",
       "evidence": "Table 4 shows year-by-year distribution: prompting dominated in 2022-2023 (8 of 14 papers), while procedural (8) and agentic (10) systems together account for 18 of 29 papers in 2025. (Section 3.4, Answer to RQ1)",
       "supported": "strong"
     },
     {
       "claim": "The unified four-paradigm taxonomy (fine-tuning, prompting, procedural, agentic) with orthogonal retrieval and analysis augmentation layers organizes the LLM-based software repair design space more precisely than prior surveys.",
       "evidence": "Table 8 compares nine prior surveys and identifies three specific gaps: no unified design space treating all four as first-class paradigms, no orthogonal augmentation modeling, and no cell-wise benchmark comparison. The paper maps 62 systems into this framework with Table 3 and Table 9. (Section 9)",
       "supported": "moderate"
     },
     {
       "claim": "Reported repair rates are shaped as much by protocol design (fault localization assumptions, pass@k budgets, benchmark subsets) as by model quality.",
       "evidence": "Tables 5-7 record evaluation assumptions alongside scores, showing that systems evaluate on different subsets of Defects4J, use varying pass@k budgets (pass@1 to pass@5000), and make different fault localization assumptions (perfect FL vs. end-to-end). The answer to RQ3 makes this point explicit. (Section 8)",
       "supported": "strong"
     },
     {
       "claim": "Four bottlenecks remain for semantic correctness and repository-scale repair: shallow test suites, expensive multi-step workflows, noisy retrieval, and weak patch assurance.",
       "evidence": "Section 10 discusses each bottleneck with specific references to surveyed systems: data leakage and weak test suites inflating scores, token and compute costs in agentic workflows, embedding-based retrieval missing key files, and LLM-generated patches introducing latent bugs. (Section 10, Answer to RQ4)",
       "supported": "moderate"
     },
     {
       "claim": "Two authors independently screened 462 records with disagreements resolved through discussion and a third author consulted for borderline cases, using Cohen's kappa for inter-rater agreement.",
       "evidence": "Section 2.4 states this procedure but the actual Cohen's kappa value and detailed disagreement statistics are deferred to the replication package rather than reported in the paper.",
       "supported": "weak"
     }
   ],
   "methodology_tags": [
     "meta-analysis"
   ],
   "key_findings": "This systematic survey proposes a unified taxonomy of 62 LLM-based software repair systems organized along two axes: control authority (who steers the repair loop) and parameter adaptation (whether the base model is fine-tuned or frozen), yielding four paradigms (fine-tuning, prompting, procedural, agentic) with retrieval and analysis augmentation as orthogonal layers. The survey finds that adoption has shifted from single-turn prompting toward procedural and agentic workflows, and that reported repair rates are heavily influenced by evaluation protocol choices (benchmark subsets, pass@k budgets, fault localization assumptions), making cross-paper comparisons unreliable without accounting for these differences. The paper consolidates benchmark trends across Defects4J, HumanEval-Java, and SWE-bench, revealing that progress on repository-level repair is closely tied to better context management and feedback rather than model scaling alone.",
   "red_flags": [
     {
       "flag": "Authors survey their own systems",
       "detail": "Several authors (Boyang Yang, Haoye Tian, and others) are creators of multiple surveyed systems including CREF [126], KGCompass [127], MORepair [128], LANTERN [79], and Luo et al. [78]. At least 5 of the 62 surveyed papers are authored by members of the survey team. This potential conflict of interest is not disclosed or discussed."
     },
     {
       "flag": "Inter-rater agreement statistics deferred to replication package",
       "detail": "The paper states that Cohen's kappa was measured for the screening process but does not report the actual value in the paper. The coding guidelines and detailed statistics are deferred to the replication package, making it impossible to assess screening reliability from the paper alone."
     },
     {
       "flag": "Survey does not perform quality assessment of individual studies",
       "detail": "While the paper records 'basic quality indicators' for each paper (mentioned in Section 2.3 and 2.4), it does not systematically assess the methodological quality of the 62 surveyed studies. The survey summarizes claimed results at face value without evaluating whether reported numbers are credible or methodologically sound, which risks laundering weak results."
     },
     {
       "flag": "No funding or competing interests disclosure",
       "detail": "The paper contains no acknowledgments section, no funding disclosure, and no competing interests statement, despite the authors being affiliated with multiple universities and having published several of the surveyed systems."
     }
   ],
   "cited_papers": [
     {
       "title": "Agentless: Demystifying LLM-Based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Key procedural pipeline for repository-level repair, achieving competitive results on SWE-bench without agentic autonomy."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Foundational tool-augmented agent for software repair using ReAct loop with GPT-4 on SWE-bench."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "The primary repository-level repair benchmark used to evaluate agentic and procedural repair systems."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Tool-augmented agent using AST navigation and spectrum-based fault localization for SWE-bench repair."
     },
     {
       "title": "ChatRepair: Automated Program Repair via Conversation",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Pioneering test-in-the-loop procedural approach demonstrating cost-effective conversational repair with ChatGPT."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2025,
       "relevance": "Agent-based repair system using finite-state controller with tool access, achieving state-of-the-art on Defects4J."
     },
     {
       "title": "SWE-RL: Advancing LLM Reasoning via Reinforcement Learning on Open Software Evolution",
       "authors": ["Yuxiang Wei", "Olivier Duchenne", "Jade Copet"],
       "year": 2025,
       "relevance": "Demonstrates RL fine-tuning on real GitHub evolution traces for repository-level repair, state-of-the-art among open-source LLMs on SWE-bench Verified."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "relevance": "Open platform for tool-augmented agentic software development and repair on SWE-bench."
     },
     {
       "title": "Abstain and Validate: A Dual-LLM Policy for Reducing Noise in Agentic Program Repair",
       "authors": ["José Cambronero", "Michele Tufano", "Sherry Shi"],
       "year": 2025,
       "arxiv_id": "2510.03217",
       "relevance": "Industrial-scale LLM-as-judge approach with abstention and validation policies for production repair at Google."
     },
     {
       "title": "MAGIS: LLM-Based Multi-Agent Framework for GitHub Issue Resolution",
       "authors": ["Wei Tao", "Yucheng Zhou", "Yanlin Wang", "Wenqiang Zhang", "Hongyu Zhang", "Yu Cheng"],
       "year": 2024,
       "relevance": "Self-controlled multi-agent system with role separation for repository-level repair on SWE-bench."
     },
     {
       "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["André Silva", "Sen Fang", "Martin Monperrus"],
       "year": 2025,
       "relevance": "Demonstrates parameter-efficient fine-tuning (LoRA) for program repair with competitive results at low compute cost."
     },
     {
       "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yang Xie", "YuXiang Ma", "Weisong Sun", "Yun Yang", "Zhenyu Chen"],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Most closely related prior survey classifying LLM-based repair by utilization modes, which this survey aims to supersede with a control-paradigm taxonomy."
     }
   ]
 }