ai-research-survey

scan.json (23121B)

---

 {
   "paper": {
     "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
     "authors": ["Quanjun Zhang", "Chunrong Fang", "Yang Xie", "Yuxiang Ma", "Weisong Sun", "Yun Yang", "Zhenyu Chen"],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2405.01466",
     "doi": "10.48550/arXiv.2405.01466"
   },
   "scan_version": 2,
   "active_modules": ["survey_methodology"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository at https://github.com/iSEngLab/AwesomeLLM4APR containing artifacts and bibliographic information for all included papers."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package repository includes all collected papers and their bibliographic information, as stated in Section 4.5: 'All included papers and their bibliographic information are publicly available in our replication repository.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or dependency specifications are provided. The paper is a survey with no computational experiments requiring environment setup, but analysis scripts (if any) have no documented environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided for replicating the search, filtering, or quality assessment process beyond the methodology description in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that reports counts and proportions of papers, not experimental results requiring confidence intervals."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The survey reports descriptive statistics (counts, percentages, distributions) without making comparative claims requiring significance tests."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental comparisons are made; the paper summarizes and categorizes existing literature."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "As a systematic review, the sample size is determined by the search and filtering process, not by experimental design requiring power analysis."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experimental runs are conducted; the paper is a literature survey."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 (Related Work) explicitly compares against prior surveys: Zhang et al. [294] on LLM4SE, Wang et al. [231] on LLM testing, Fan et al. [49] on LLMs in SE, Hou et al. [83] on LLM4SE SLR, Gazzola et al. [66] and Monperrus et al. [163] on APR, and Zhang et al. [292] on learning-based APR."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The related surveys cited are contemporary (2023-2024), including Zhang et al. [294], Hou et al. [83], and Wang et al. [231], all from 2023-2024."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "As a systematic literature review, there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not run experiments with metrics; it categorizes and summarizes literature."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation of system outputs is not applicable to a literature survey."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No test set is used; this is a systematic review."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides extensive breakdowns: by year (Fig. 3), venue (Table 4), programming language (Fig. 4), contribution type (Table 5), LLM type (Table 6), utilization strategy (Table 7), bug type (Table 8), dataset (Table 13), and input form (Table 14)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 discusses challenges and limitations of the field including data leakage, cost issues, limited human studies, and under-explored repair scenarios."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative findings: 53.85% of novel approaches fail to release artifacts (Section 8.5), Olausson et al. finding self-repair is 'not a panacea' (Section 8.4), and limited human studies in the field (only 3 papers)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims to analyze 189 papers from LLMs, APR, and integration perspectives, which is supported by the detailed analysis across Sections 5-8. Claims about categorizing LLMs, repair scenarios, and integration factors are all substantiated."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a systematic review that describes trends and categorizes work without making causal claims. Statements like 'LLMs have brought significant changes' are descriptive observations about the literature, not causal inferences."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its scope to 2020-September 2025 (Section 4), to specific search databases (Google Scholar, ACM DL, IEEE Explore), and to papers meeting their inclusion/exclusion criteria. Section 10.3 discusses external validity limitations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Threats to Validity) discusses construct, internal, and external validity threats including potential bias in paper selection, keyword limitations, and generalizability concerns."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match its measurements — it claims to summarize and categorize 189 papers, and that is what it does. It does not frame paper counts as proxies for broader phenomena without acknowledgment."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any LLMs in its own methodology; it is a survey reviewing papers that use LLMs."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used in the survey methodology."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No models are run; this is a literature survey."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used in this survey."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents its filtering pipeline in detail: 7,253 initial papers → 285 after inclusion/exclusion criteria → 167 after quality assessment → 189 after snowballing (Sections 4.3-4.5, Fig. 2). Criteria are specified in Tables 2 and 3."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Threats to Validity) provides a dedicated discussion of construct validity (10.1), internal validity (10.2), and external validity (10.3)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 discusses specific threats: ambiguity in paper categorization (10.1), potential bias from two-author review process (10.2), limitations of keyword search that may miss papers (10.3), and the rapidly evolving nature of the field."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states its scope: papers from 2020 to September 2025, specific search databases, English-language papers only, and papers meeting their quality threshold of 8/10 points. Section 10.3 acknowledges papers may have been missed."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at https://github.com/iSEngLab/AwesomeLLM4APR provides the full list of included papers and bibliographic information, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data collection in detail: manual search of 6 SE venues yielding 25 seed papers, automated search across 3 digital libraries yielding 7,253 papers, and snowballing search adding 21 papers."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved; data sources are standard digital libraries and venues which are fully described."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Fig. 2 and Sections 4.3-4.5 document the full pipeline: manual search (25 papers) → automated search (7,253) → inclusion/exclusion filtering (285) → quality assessment (167) → snowballing (189), with counts at each stage and criteria described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section discloses funding from the Australian Research Council (DP200102491, DP230101790), National Natural Science Foundation of China (61932012, 62141215, 62372228), Natural Science Foundation of Jiangsu Province, and Fundamental Research Funds."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Nanjing University of Science and Technology, Nanjing University, Nanyang Technological University, and Swinburne University of Technology."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are government research councils and university funds (Australian Research Council, NSFC, Jiangsu Province) with no financial stake in the survey's conclusions about LLM-based APR."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a systematic literature review that does not evaluate any pre-trained model on a benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is performed in this survey."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No model evaluation is performed in this survey."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this systematic literature review."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method to report costs for."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": true,
         "justification": "The paper follows SEGRESS guidelines (Section 4), uses the Quasi-Gold Standard (QGS) method for search string construction, and documents a structured multi-stage search process with manual search, automated search, and snowballing (Fig. 2). Inclusion/exclusion criteria (Table 2) and quality assessment criteria (Table 3) are formally specified."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.2 describes a 10-question quality assessment checklist (Table 3) where two authors independently rate papers on a three-tier scale (yes/partial/no). Papers scoring below 8 out of 10 are excluded. This reduced the pool from 285 to 167 papers."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss publication bias. There is no funnel plot, no discussion of negative-result underrepresentation, and no acknowledgment that the published LLM-based APR literature likely skews toward positive results."
       }
     }
   },
   "claims": [
     {
       "claim": "189 relevant LLM-based APR papers were published between 2020 and September 2025, showing a rapidly increasing trend.",
       "evidence": "Section 5.1 and Fig. 3 show the publication trend: 1 (2020), 5 (2021), 13 (2022), 53 (2023), 66 (2024), 51 (2025 partial). Power function fit achieves R²=0.98833.",
       "supported": "strong"
     },
     {
       "claim": "GPT-3.5 and GPT-4 are the two most popular LLMs in APR research, used in 72 and 51 papers respectively.",
       "evidence": "Table 6 provides detailed year-by-year usage counts for 78 LLMs across all 189 papers.",
       "supported": "strong"
     },
     {
       "claim": "LLM-based APR has been applied to 20 different bug types and 24 programming languages.",
       "evidence": "Table 8 lists all 20 bug types with paper counts, and Fig. 4 shows the distribution across 24 programming languages.",
       "supported": "strong"
     },
     {
       "claim": "62.99% of collected papers provide publicly available artifacts, rising to 86.84% for top-tier SE venues.",
       "evidence": "Section 8.5 states '80 studies provide the replication packages in their papers, accounting for 62.99% (80/189)' and '86.84% of papers (33/38) make related artifacts publicly open' for top venues.",
       "supported": "strong"
     },
     {
       "claim": "Semantic bugs constitute approximately 42.93% of LLM-based APR research, making it the dominant repair scenario.",
       "evidence": "Table 8 shows semantic bugs with the highest paper count across all years, with explicit percentage stated in Section 7.",
       "supported": "strong"
     },
     {
       "claim": "There is a shift from fine-tuning toward zero-shot prompting and agent-based approaches as LLMs have grown in capability.",
       "evidence": "Table 7 shows fine-tuning dominated early years (2021-2022) while zero-shot and agent approaches emerged in 2023-2025. Section 6.2.5 discusses this trend.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This systematic review of 189 LLM-based APR papers (2020-2025) identifies 78 distinct LLMs used for program repair, with GPT-3.5 (72 papers) and GPT-4 (51 papers) dominating. The field shows a clear evolution from fine-tuning smaller models toward zero-shot prompting and agent-based approaches with larger models. LLM-based APR has expanded to 20 bug types and 24 programming languages, far exceeding traditional APR's scope. Open science remains a concern, with 53.85% of novel approaches failing to release artifacts, though top-tier venues show 86.84% availability.",
   "red_flags": [
     {
       "flag": "No publication bias analysis",
       "detail": "The survey does not discuss whether the LLM-based APR literature it reviews is biased toward positive results. Papers showing LLMs fail at APR tasks are less likely to be published, which could inflate the apparent effectiveness of the field."
     },
     {
       "flag": "Quality assessment lacks inter-rater reliability reporting",
       "detail": "While two authors independently performed quality assessment, the paper does not report inter-rater agreement statistics (e.g., Cohen's kappa) for the 10-question quality assessment, only stating disagreements were resolved through discussion."
     },
     {
       "flag": "No critical synthesis of methodological quality",
       "detail": "Despite having a quality assessment checklist, the paper does not report the aggregate quality scores or analyze how methodological quality varies across the 189 papers. It treats all papers that passed the threshold equally, potentially laundering weak results alongside strong ones."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Premier benchmark for evaluating LLM agents on real-world repository-level software engineering tasks."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Key agentic approach for repository-level program repair using LLM-based agents with tool interfaces."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open platform for AI coding agents with generalist capabilities for software development tasks."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Comprehensive empirical study of 9 LLMs across 5 datasets for program repair, establishing key findings about LLM capabilities."
     },
     {
       "title": "A Survey on Large Language Models for Software Engineering",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yang Xie"],
       "year": 2023,
       "arxiv_id": "2312.15223",
       "relevance": "Broader survey on LLMs in SE providing context for the APR-specific review."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "First autonomous LLM-based repair agent using ChatGPT with state machine for iterative debugging."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "AST-based repository-level repair agent demonstrating autonomous program improvement."
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 Each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Pioneering conversational APR approach (ChatRepair) with cost analysis, demonstrating practical LLM-based repair."
     },
     {
       "title": "Is Self-Repair a Silver Bullet for Code Generation?",
       "authors": ["Theo X. Olausson", "Jeevana Priya Inala", "Chenglong Wang", "Jianfeng Gao", "Armando Solar-Lezama"],
       "year": 2024,
       "relevance": "Empirical study questioning the effectiveness of LLM self-repair, finding existing models often fail to provide reliable feedback."
     },
     {
       "title": "A Survey of Learning-Based Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yuxiang Ma", "Weisong Sun", "Zhenyu Chen"],
       "year": 2023,
       "relevance": "Predecessor survey on learning-based APR that this work directly extends to the LLM era."
     },
     {
       "title": "SWE-RL: Advancing LLM Reasoning via Reinforcement Learning on Open Software Evolution",
       "authors": ["Yuxiang Wei"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "RL-based approach to improving LLM reasoning for software engineering tasks."
     },
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2025,
       "relevance": "Procedural approach to repository-level repair without agentic scaffolding, providing important baseline."
     }
   ]
 }