ai-research-survey

scan.json (24303B)

---

 {
   "paper": {
     "title": "Large Language Models Meet Automated Program Repair: Innovations, Challenges and Solutions",
     "authors": ["Yiting Tang"],
     "year": 2024,
     "venue": "Proceedings of the 2nd International Conference on Machine Learning and Automation",
     "doi": "10.54254/2755-2721/113/2024.18303"
   },
   "scan_version": 3,
   "active_modules": ["survey_methodology"],
   "methodology_tags": ["meta-analysis"],
   "key_findings": "This survey reviews 8 LLM-based automated program repair systems (FitRepair, TypeFix, InferFix, RepairAgent, FixAgent, ThinkRepair, SRepair, ChatRepair), comparing their model architectures, techniques, and benchmark performance. The paper argues that LLM-based APR surpasses traditional neural program repair in performance and generality, particularly in zero-shot scenarios. It identifies data leakage and high computational overhead as key open challenges and proposes solution directions including data filtering, memorization detection, candidate patch reduction, and self-correction techniques.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code, analysis scripts, or repository links are provided. A survey can release its analysis code and collected data but this paper does not."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset, spreadsheet of reviewed systems, or structured data from the review is released. The comparison data in Table 1 is the extent of structured information and is only in the paper body."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment or tooling specifications are provided. This is a narrative survey with no computational component described."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No instructions for reproducing the review are provided — no search queries, databases searched, or systematic process to follow."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments. The paper reports numbers from other papers' results (Table 1) but does not conduct its own statistical analyses."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments or meta-analytic statistical aggregation."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments or statistical aggregation."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to be 'the first to provide a comprehensive review' of LLM-based APR but does not compare its coverage or methodology against any prior surveys on APR or related topics."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No prior surveys are cited or compared against to establish how this review advances the state of knowledge."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system with components to ablate — this is a survey paper."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No original experiments conducted."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted that would warrant human evaluation."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a per-system breakdown across model name, architecture, technique, and performance on multiple benchmarks (Defects4J, HumanEval-Java, QuixBugs, BugsInPy, CodeFlaw)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses challenges including data leakage (Section 5.1) and high computational overhead (Section 5.3) as failure modes of LLM-based APR."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports negative aspects including data leakage concerns, high fine-tuning costs, lack of domain-specific knowledge, and insufficient adaptability in complex repair scenarios (Sections 5.1, 5.3, and the abstract)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims 'our paper is the first to provide a comprehensive review of the LLM-based APR domain' but the paper reviews only 8 systems with very brief descriptions (1-3 sentences each in Sections 3.1-3.8). The claim of comprehensiveness is not supported by the shallow coverage. The claim that 'zero-shot LLM-based APRs has surpassed that of NPR' is stated without systematic evidence — Table 1 has mostly empty cells making comparison impossible."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a narrative survey that summarizes existing work. It does not make original causal claims requiring experimental justification."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes broad claims like 'LLMs have demonstrated advantages over traditional NPR approaches in various repair scenarios' and 'LLM-based APR excels in repair performance and flexibility' but only reviews 8 systems tested on a narrow set of Java/Python benchmarks. No scope boundaries are stated for these generalizations."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider alternative explanations for why LLM-based APR outperforms NPR (e.g., differences in training data scale, evaluation methodology differences across studies, or benchmark-specific advantages)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates benchmark fix counts (e.g., bugs fixed on Defects4J) with 'repair performance' and 'software reliability' without discussing the gap between fixing curated benchmark bugs and actual software repair quality in real-world settings."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use any models directly. Model names are listed for reviewed systems but the survey itself has no experimental setup."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use prompting."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no experimental setup requiring hyperparameters."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not use agentic scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document any paper selection pipeline — no search queries, databases searched, inclusion/exclusion criteria, or filtering steps. The 8 systems appear to have been selected ad hoc with no stated methodology."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion (Section 6) mentions future directions but does not discuss limitations of the review itself."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not acknowledge potential issues with its own review methodology, scope, or selection bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what is excluded from the review. There are no statements about which types of APR systems, time periods, or venues were out of scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data or supplementary materials are provided. The comparison data in Table 1 is the only structured data and appears to be manually compiled with no verification possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the 8 APR systems were identified or selected for review. No search strategy, databases, or selection criteria are mentioned."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline is documented. The paper provides no information about how papers were found, screened, or how performance numbers in Table 1 were extracted and verified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation with Northwest Minzu University, Lanzhou, China is clearly stated below the author name."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so funder independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper that does not evaluate any pre-trained model on benchmarks. The paper does discuss contamination as a challenge for reviewed systems (Section 5.1) but does not itself perform benchmark evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this survey paper."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no original method or experiments. The paper discusses costs of reviewed systems but has no cost of its own."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "Survey paper with no computational experiments."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": true,
         "answer": false,
         "justification": "No PRISMA flow diagram, no structured search strategy, no reproducible queries, and no review protocol. The 8 systems appear selected without any stated methodology."
       },
       "quality_assessment_of_sources": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not assess the methodological quality of the 8 reviewed systems. It reports their claimed performance numbers from Table 1 without evaluating whether those results are trustworthy, comparable, or methodologically sound."
       },
       "publication_bias_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of publication bias. The paper does not consider whether the reviewed systems represent a biased sample of positive results or whether negative results in APR are underrepresented."
       }
     }
   },
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Provides a brief overview of 8 APR tools that practitioners might find useful as an entry point, but offers no actionable guidance or new tools."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected narrative that LLMs outperform traditional NPR methods, with no surprising findings."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward positive framing of LLM-based APR progress."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Unknown venue (ICMLA), single author from a regional university, no association with prominent labs."
     }
   },
   "claims": [
     {
       "claim": "LLM-based APR exhibits superior repair performance and enhanced generality compared to traditional NPR methods.",
       "evidence": "Stated in abstract and Section 1. Table 1 shows performance numbers for 8 systems on various benchmarks, but most cells are empty (dashes), making systematic comparison impossible.",
       "supported": "weak"
     },
     {
       "claim": "Zero-shot LLM-based APR has surpassed NPR performance.",
       "evidence": "Stated in abstract. Section 4.1 mentions AlphaRepair as a zero-shot approach but provides no direct comparative data between zero-shot LLM-APR and NPR on the same benchmarks.",
       "supported": "unsupported"
     },
     {
       "claim": "This is the first comprehensive review of LLM-based APR from perspectives of innovation, challenges, and solutions.",
       "evidence": "Stated in contributions (Section 1). No search of prior surveys is conducted to verify this novelty claim. The review covers only 8 systems with 1-3 sentence descriptions each.",
       "supported": "unsupported"
     },
     {
       "claim": "Data leakage and high computational overhead are key challenges for LLM-based APR.",
       "evidence": "Section 5 discusses these challenges with references to relevant work (Yang et al. 2024 on memorization, Zhong et al. 2022 on reducing candidate patches). The challenges are well-supported by the cited literature.",
       "supported": "moderate"
     },
     {
       "claim": "Reducing candidate patches to approximately 170 achieves 90% of optimal performance.",
       "evidence": "Section 5.4.1 cites StandUp4NPR (Zhong et al. 2022) for this claim. The number comes from the cited paper, not original analysis.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No systematic review methodology",
       "detail": "The paper claims to be a 'comprehensive review' but has no PRISMA protocol, no search strategy, no inclusion/exclusion criteria, and no stated methodology for selecting the 8 reviewed systems. The selection appears ad hoc."
     },
     {
       "flag": "Extremely shallow coverage despite comprehensiveness claim",
       "detail": "Each of the 8 APR systems receives only 1-3 sentences of description (Sections 3.1-3.8). FitRepair, TypeFix, InferFix, RepairAgent, and FixAgent each get a single short paragraph with no technical depth. This does not constitute a comprehensive review."
     },
     {
       "flag": "Table 1 is mostly empty",
       "detail": "The main comparison table (Table 1) has dashes (-) in the majority of performance cells, making cross-system comparison impossible. For example, InferFix shows no results on any benchmark, and most systems are missing results on 4 of 5 benchmarks."
     },
     {
       "flag": "No quality assessment of reviewed papers",
       "detail": "The survey reports performance numbers from reviewed papers at face value without assessing their methodological quality, whether results are comparable across different evaluation setups, or whether claimed numbers are trustworthy. This launders the signal-to-noise ratio of the sources."
     },
     {
       "flag": "Duplicate and inconsistent references",
       "detail": "InferFix appears as references [8], [15], and [21] — the same paper cited three times with the same text. SRepair (Wu et al. 2024) appears as [19], [29], [32], and [35]. This suggests careless reference management."
     },
     {
       "flag": "Author name discrepancy",
       "detail": "The paper header lists 'Yiting Tang' but the email prefix shows 'tangyt' — minor, but the registry entry lists 'Yi Tang' which differs from the paper itself."
     },
     {
       "flag": "Overclaiming novelty",
       "detail": "The paper claims to be 'the first to provide a comprehensive review' of LLM-based APR, but does not cite or compare against existing surveys on APR (e.g., Zhong et al. 2022 'Neural Program Repair: Systems, Challenges, and Solutions' is reference [7] and covers overlapping ground)."
     }
   ],
   "cited_papers": [
     {
       "title": "Cure: Code-Aware Neural Machine Translation for Automated Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "doi": "10.1109/ICSE43902.2021.00107",
       "relevance": "Neural machine translation approach to APR, demonstrates code-aware techniques for automated bug fixing."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Introduces AlphaRepair, demonstrating that zero-shot LLM-based repair can surpass trained NPR systems, key milestone in APR evolution."
     },
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "ChatRepair — demonstrates conversational LLM-based APR with multi-round dialogue, reports cost-effectiveness metrics."
     },
     {
       "title": "Revisiting the Plastic Surgery Hypothesis via Large Language Models",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2303.10494",
       "relevance": "FitRepair — uses model ensemble with retrieval-enhanced CodeT5 for program repair."
     },
     {
       "title": "InferFix: End-to-end program repair with LLMs over Retrieval-augmented prompts",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano", "Xin Shi", "Shuai Lu", "Neel Sundaresan", "Alexey Svyatkovskiy"],
       "year": 2023,
       "relevance": "Retrieval-augmented LLM approach to APR using Codex, demonstrates industry-scale program repair."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous agent-based APR using GPT-3.5, represents the shift toward agentic program repair systems."
     },
     {
       "title": "ThinkRepair: Self-Directed Automated Program Repair",
       "authors": ["Xin Yin", "Chao Ni", "Shaohua Wang", "Zhenhao Li", "Limin Zeng", "Xiaohu Yang"],
       "year": 2024,
       "doi": "10.1145/3650212.3680359",
       "relevance": "Uses Chain-of-Thought reasoning for APR, demonstrating advanced prompting techniques for bug repair."
     },
     {
       "title": "A Unified Debugging Approach via LLM-Based Multi-Agent Synergy",
       "authors": ["Cheryl Lee", "Chunqiu Steven Xia", "Jen-tse Huang", "Zhouruixin Zhu", "Lingming Zhang", "Michael R. Lyu"],
       "year": 2024,
       "relevance": "FixAgent — multi-agent system for debugging combining multiple repair strategies with GPT-4."
     },
     {
       "title": "How Far Can We Go with Practical Function-Level Program Repair?",
       "authors": ["Mingyuan Wu", "Jiahong Xiang", "Xiaoyang Xu", "Fanchu Kong", "Haotian Zhang", "Yuqun Zhang"],
       "year": 2024,
       "arxiv_id": "2404.12833",
       "relevance": "SRepair — function-level APR using GPT-3.5 + MigiCoder-7B pipeline, addresses practical scalability of LLM-based repair."
     },
     {
       "title": "Domain Knowledge Matters: Improving Prompts with Fix Templates for Repairing Python Type Errors",
       "authors": ["Yun Peng", "Shuzheng Gao", "Cuiyun Gao", "Yintong Huo", "Michael R. Lyu"],
       "year": 2023,
       "relevance": "TypeFix — domain-specific APR for Python type errors using retrieval and fine-tuning on CodeT5."
     },
     {
       "title": "Unveiling Memorization in Code Models",
       "authors": ["Zhou Yang", "Zhipeng Zhao", "Chenyu Wang", "Jieke Shi", "Dongsun Kim", "Donggyun Han", "David Lo"],
       "year": 2024,
       "doi": "10.1145/3597503.3639074",
       "relevance": "Directly relevant to data contamination and leakage detection in code LLMs, proposes Type-1 clone detection and perplexity-based memorization detection."
     },
     {
       "title": "StandUp4NPR: Standardizing SetUp for Empirically Comparing Neural Program Repair Systems",
       "authors": ["Wenkang Zhong", "Hongliang Ge", "Hongfei Ai", "Chuanyi Li", "Kui Liu", "Jidong Ge", "Bin Luo"],
       "year": 2022,
       "doi": "10.1145/3551349.3556943",
       "relevance": "Standardizes NPR evaluation methodology, provides empirical findings on candidate patch reduction for cost optimization."
     }
   ]
 }