ai-research-survey

scan.json (23729B)

---

 {
   "paper": {
     "title": "Automated Program Repair: Emerging trends pose and expose problems for benchmarks",
     "authors": [
       "Joseph Renzullo",
       "Pemma Reiter",
       "Westley Weimer",
       "Stephanie Forrest"
     ],
     "year": 2024,
     "venue": "arXiv preprint (submitted to ACM)",
     "arxiv_id": "2405.05455",
     "doi": "XXXXXXX.XXXXXXX"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or data release is mentioned anywhere in the paper. The survey's corpus of 118 papers and analysis data are not publicly available."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not release its corpus of 118 reviewed papers, keyword analysis data, or any structured dataset. A survey can release its extracted data and analysis scripts, but this one does not."
       },
       "environment_specified": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey/review paper with no computational experiments requiring an environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. While Section 1.1 describes the methodology (keyword search across five venues, 2018-2023), there are no step-by-step instructions, search queries, or scripts that would allow someone to replicate the paper selection process."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative survey that does not report quantitative experimental results requiring confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not make comparative statistical claims that would require significance tests. It is a qualitative topical review."
       },
       "effect_sizes_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments are conducted; the paper is a qualitative survey reviewing trends and methodological issues in APR literature."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "No experimental sample sizes are involved. The corpus of 118 papers is described but this is a literature review, not a statistical study."
       },
       "variance_reported": {
         "applies": false,
         "answer": false,
         "justification": "No quantitative experiments are run, so variance across runs is not applicable."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": false,
         "answer": false,
         "justification": "This is a topical review, not a system or method evaluation. There is no system to compare against baselines."
       },
       "baselines_contemporary": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is performed; the paper is a survey of the APR field."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "No system or method is proposed, so ablation is not applicable."
       },
       "multiple_metrics": {
         "applies": false,
         "answer": false,
         "justification": "No system evaluation is performed; the paper reviews how other papers report metrics but does not evaluate a system itself."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "No system outputs are produced that would require human evaluation. The paper is a survey."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with training/test splits are conducted."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides a per-keyword breakdown of the 118 papers. The paper also breaks down findings by category: ML architectures (Section 3), data leakage issues (Section 4), and benchmark problems (Section 5), with specific counts (e.g., 29% use ML keywords, 44 papers evaluate on Defects4J vs. 5 on ManyBugs)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failure cases in the reviewed literature: inconsistent correctness definitions (Section 1), papers that do not consider data leakage (Section 4.2 names five specific papers), tools that cannot be compared due to incompatible reporting, and benchmark limitations throughout Section 5."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central thesis is a negative finding: current APR benchmarks and evaluation practices are inadequate for ML-based approaches. It reports specific negative observations: 5 papers ignore data leakage, vocabulary sizes are not always reported, decontamination is often infeasible, and StandUp4NPR has design limitations (Section 6.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that ML pervades APR, that evaluations must take care to ensure validity, and that popular benchmarks were not designed with ML in mind. All three are supported by the detailed analysis in Sections 3-5, including the keyword analysis (Table 1) and specific benchmark issues documented throughout."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It describes trends and identifies methodological problems in the literature but does not claim that X causes Y. Statements like 'Duplicated data lead to bias' reference general ML principles, not original causal findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 clearly bounds the scope: 'we take a narrower view, examining all of the papers published at the top five software engineering venues since 2018 and focusing on ML in APR.' The five venues are explicitly named (ICSE, TSE, FSE, ASE, EMSE) and the time range specified (Jan 2018 - Sep 2023)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its observations. For example, the concentration on Defects4J (44 papers) vs. ManyBugs (5 papers) is attributed partly to build complexity, but the paper notes 'there are other factors that influence the differences' without systematically exploring them. No threats-to-validity or alternative-explanations section exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": false,
         "answer": false,
         "justification": "No ML models are used in this survey. The paper reviews others' use of models but does not run any models itself."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "No prompting is used. This is a survey paper."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No experiments with hyperparameters are conducted."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Section 1.1 describes the methodology at a high level: seeding from Monperrus's living review, selecting five venues, searching by primary keywords (patch, repair) and secondary keywords (automate, fix, fault, bug, vulnerability, generate), and manually reviewing abstracts. However, the actual filtering criteria for what counted as APR-related during abstract review are not stated, nor are counts given at each filtering stage. The paper states 118 papers were identified and 65 cross-listed with Monperrus, but does not document how many were found at each stage of screening."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper discusses limitations of other work but does not have a section discussing limitations of its own review methodology."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed for this review itself. Issues like selection bias (only five venues), time-boundedness (2018-2023), keyword-based search potentially missing relevant papers, and the subjective abstract screening process are not addressed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 explicitly states the scope: 'we take a narrower view, examining all of the papers published at the top five software engineering venues since 2018 and focusing on ML in APR.' The venues and time period are specified. However, the paper does not explicitly state what the results do NOT show, just what was covered."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The corpus of 118 papers and the extracted metadata (keywords, ML techniques, benchmarks used, etc.) are not made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 1.1 describes the data collection procedure: seeding from Monperrus's living review, identifying top five venues by volume, searching with primary and secondary keywords, manually reviewing abstracts, covering January 2018 through September 2023. The total of 118 papers and overlap with the living review (65) are reported."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data source is published papers from known venues."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the high-level pipeline (living review seed → venue selection → keyword search → abstract review → 118 papers) but does not document intermediate counts at each stage. How many papers were returned by keyword search before abstract filtering? How many were excluded at each stage and why? These details are missing."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Renzullo and Reiter at Arizona State University, Weimer at University of Michigan, Forrest at Arizona State University (also Santa Fe Institute). These are academic institutions with no apparent conflict regarding the APR tools reviewed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of funding disclosure is the issue, not a known conflict."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Note that Weimer and Forrest are co-authors of GenProg, which is discussed in the paper (Section 6.1), but this is not disclosed as a potential conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "This paper does not evaluate any pre-trained model on a benchmark. It is a survey reviewing how others handle contamination."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any model itself. It reviews contamination practices in the literature."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The paper surveys contamination risks but does not run its own model evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this survey."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational method whose cost needs reporting."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "This is a survey paper with no computational experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "ML techniques now pervade APR, with more than three in four papers published by 2023 employing ML techniques, up from fewer than one in six in 2018.",
       "evidence": "Section 3 and Table 1 report keyword analysis of 118 papers across five top SE venues (2018-2023), showing 29% list ML-related keywords overall with a sharp upward trend.",
       "supported": "moderate"
     },
     {
       "claim": "Five of the surveyed papers that use LLMs for APR do not consider the impact of data leakage and contamination.",
       "evidence": "Section 4.2 names the five specific papers: [4, 34, 37, 93, 120] (SynShine, Fan et al., VulRepair, Rete, and Xia & Zhang 2022).",
       "supported": "strong"
     },
     {
       "claim": "APR benchmarks are too small to train ML tools and systematically differ from ML datasets in construction and purpose.",
       "evidence": "Section 5.4 argues that APR benchmarks consist of limited bug/fix pairs with test cases, while ML datasets must be orders of magnitude larger and lack execution environments. Specific examples cited include CrossVul and CURE's training data.",
       "supported": "strong"
     },
     {
       "claim": "Defects4J is far more widely adopted than ManyBugs, with 44 papers in the corpus evaluating on Defects4J vs. 5 on ManyBugs.",
       "evidence": "Section 5.6 provides explicit counts: 44 papers for Defects4J and 5 for ManyBugs, with references listed. The paper attributes this partly to Java's simpler build infrastructure vs. C's build complexity.",
       "supported": "strong"
     },
     {
       "claim": "NPR techniques have not converged on a standard metric for search efficiency, with beam sizes ranging from 50 to 1000.",
       "evidence": "Section 5.7 documents beam sizes across tools: 50 (VRepair, SeqTrans), 200 (RewardRepair), 1000 (Knod, Rete), and notes that generative vocabulary size is not always reported, making search efficiency comparisons difficult.",
       "supported": "strong"
     },
     {
       "claim": "StandUp4NPR, while promising for standardization, has design limitations that prevent fair comparison of traditional APR algorithms.",
       "evidence": "Section 6.1 explains that StandUp4NPR assumes all patches are generated simultaneously and evaluated independently, which prevents search-based techniques like GenProg and ARJA from using evaluation feedback loops.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "meta-analysis",
     "qualitative"
   ],
   "key_findings": "ML techniques have rapidly taken over APR research, rising from fewer than 1-in-6 papers in 2018 to more than 3-in-4 by 2023 across top SE venues. The paper identifies three critical methodological problems: (1) data leakage and contamination are inadequately addressed, with five LLM-based APR papers ignoring the issue entirely; (2) popular benchmarks like Defects4J were not designed for ML evaluation and are too small for training; (3) inconsistent reporting of correctness criteria, fault localization assumptions, and search efficiency metrics makes cross-study comparison infeasible. The authors discuss StandUp4NPR and APR-COMP 2024 as partial solutions but note both have significant design limitations.",
   "red_flags": [
     {
       "flag": "No limitations section for own methodology",
       "detail": "The paper extensively critiques methodological weaknesses in the reviewed literature but does not discuss limitations of its own survey methodology. Potential issues include: coverage limited to five venues (missing workshops, arXiv-only papers), keyword-based search may miss relevant papers, and subjective abstract screening with unclear criteria."
     },
     {
       "flag": "Authors evaluate their own tool without disclosure",
       "detail": "Weimer and Forrest are co-authors of GenProg, which is discussed in Section 6.1 in the context of StandUp4NPR's limitations. While the discussion is fair, the relationship is not disclosed as a potential conflict of interest."
     },
     {
       "flag": "No data release for a survey paper",
       "detail": "The corpus of 118 papers, keyword analysis, and categorization data are not released. This makes it impossible to independently verify the paper counts, keyword trends, or the specific claims about which papers do or do not address contamination."
     },
     {
       "flag": "Incomplete filtering pipeline documentation",
       "detail": "The paper search methodology (Section 1.1) does not report how many papers were returned by keyword search before abstract screening, nor how many were excluded at each stage. The 118 final papers cannot be traced back through documented filtering steps."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Directly evaluates LLMs for APR and is one of the few papers that reports contamination information for evaluated PLMs."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Applies zero-shot LLMs to APR, cited as one of five papers that do not consider data leakage impact."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval, foundational to LLM code generation evaluation and contamination concerns."
     },
     {
       "title": "Extracting Training Data from Large Language Models",
       "authors": ["Nicholas Carlini", "Florian Tramèr", "Eric Wallace"],
       "year": 2021,
       "relevance": "Demonstrates memorization and data extraction from LLMs, directly relevant to contamination and data integrity in AI evaluations."
     },
     {
       "title": "Deduplicating Training Data Makes Language Models Better",
       "authors": ["Katherine Lee", "Daphne Ippolito", "Andrew Nystrom"],
       "year": 2022,
       "arxiv_id": "2107.06499",
       "relevance": "Shows deduplication reduces memorization in language models, directly relevant to benchmark contamination mitigation."
     },
     {
       "title": "StandUp4NPR: Standardizing SetUp for Empirically Comparing Neural Program Repair Systems",
       "authors": ["Wenkang Zhong", "Hongliang Ge", "Hongfei Ai"],
       "year": 2022,
       "relevance": "Proposes standardized evaluation framework for NPR tools, extensively discussed in Section 6.1 as a partial solution with limitations."
     },
     {
       "title": "RunBugRun – An Executable Dataset for Automated Program Repair",
       "authors": ["Julian Aron Prenner", "Romain Robbes"],
       "year": 2023,
       "arxiv_id": "2304.01102",
       "relevance": "Addresses the build reproducibility problem for ML-based APR evaluation with an executable benchmark dataset."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Describes GPT-4's decontamination approach (multi-sample substring matching), relevant to contamination methodology in LLM evaluations."
     },
     {
       "title": "MultiPL-E: A Scalable and Polyglot Approach to Benchmarking Neural Code Generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2023,
       "doi": "10.1109/TSE.2023.3267446",
       "relevance": "Multi-language code generation benchmark, relevant to expanding APR evaluation beyond Java."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00125",
       "relevance": "Evaluates the impact of code LLMs on APR, directly relevant to understanding how language models change the APR landscape."
     },
     {
       "title": "A Systematic Evaluation of Large Language Models of Code",
       "authors": ["Frank F Xu", "Uri Alon", "Graham Neubig", "Vincent Josua Hellendoorn"],
       "year": 2022,
       "relevance": "Systematic evaluation of code LLMs, relevant to understanding code duplication and evaluation methodology concerns."
     },
     {
       "title": "The Adverse Effects of Code Duplication in Machine Learning Models of Code",
       "authors": ["Miltiadis Allamanis"],
       "year": 2019,
       "doi": "10.1145/3359591.3359735",
       "relevance": "Shows near-duplicate code inflates ML performance metrics by up to 100%, directly motivating the contamination concerns in this paper."
     }
   ]
 }