ai-research-survey

scan.json (29674B)

---

 {
   "paper": {
     "title": "RelRepair: Enhancing Automated Program Repair by Retrieving Relevant Code",
     "authors": ["Shunyu Liu", "Guangdong Bai", "Mark Utting", "Guowei Yang"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2509.16701",
     "doi": "10.48550/arXiv.2509.16701"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RelRepair uses retrieval-augmented generation to enhance LLM-based automated program repair by retrieving relevant function signatures and code snippets from the project codebase. The approach fixes 101 bugs in Defects4J V1.2 (112.8% improvement over base ChatGPT) and achieves 48.3% fix rate on ManySStuBs4J (up from 31.2% baseline). Query rewriting is critical, boosting total repairs from 20 to 54 in ablation. SnipRepair (code snippet retrieval) handles complex bugs more effectively than SigRepair (function signature retrieval), but at higher computational cost.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or archive link is provided anywhere in the paper. The implementation is described (Section 5.3) but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available benchmarks: Defects4J V1.2 (Section 5.1, ref [25]) and ManySStuBs4J (ref [26]). Both are standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.3 mentions Python, gpt-3.5, SentenceBERT, and CodeBERT, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The algorithmic descriptions (Algorithms 1-3) describe the approach but not how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 101 bugs fixed, 48.3% fix rate). No confidence intervals or error bars appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RelRepair 'outperforms' baselines based solely on comparing raw counts (e.g., 101 vs 98 vs 76). No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are reported: '112.8%' improvement over BaseChatGPT, '37.2%' improvement with ChatGPT-4, fix rate 'from 31.2% to 48.3%' (Section 6.1.2, 6.2.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "For ManySStuBs4J, 30 bugs per category are selected 'to balance efficiency and representativeness' (Section 5.1), but no power analysis or statistical justification is given. For Defects4J, the standard benchmark is used without justification of sufficiency."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single experimental runs despite the stochastic nature of LLM sampling at temperature=1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against four state-of-the-art APR tools (ChatRepair, ThinkRepair, Mulpor, RAP-Gen) plus a BaseChatGPT baseline without retrieval (Section 5.2, Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ThinkRepair (2024), Mulpor (2024), ChatRepair (2023), and RAP-Gen (2023), all published within 1-2 years of this work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 6.3) provides a thorough ablation: BaseRepair vs SigRepair vs SnipRepair (Table 4), query rewriting on/off (Table 5), parameter selection analysis (Figure 8), and resource consumption analysis (Figure 7)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses essentially one metric: number of correct patches (patches passing all tests and semantically equivalent to the actual fix). Section 5.2 states 'we adopt this widely used evaluation metric: correct patches.' Fix rate is just a normalization of the same count."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 and Section 7 describe manual validation: 'we carefully examined each generated patch to ensure accurate validation' and 'manual validation to identify plausible patches that are functionally equivalent to the actual fixes.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J V1.2 and ManySStuBs4J are standard, established benchmarks not used for tuning the approach. Retrieval is done per-bug from the project codebase, not from the benchmark itself."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by project for Defects4J (Table 1, Table 3, Table 4) and by 16 bug categories for ManySStuBs4J (Table 6, Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper focuses on successes (e.g., Chart-10 example in Section 6.1.2). No systematic error analysis or discussion of where/why the approach fails is provided. The Venn diagram shows complementarity but not failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3.2 reports that SigRepair has limited effectiveness for complex bugs, query rewriting removal causes dramatic degradation (54→20 fixes, Table 5), and SigRepair saturates quickly despite increased patches (Figure 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '101 bugs in Defects4J V1.2' matches Table 1, '17.1% improvement in ManySStuBs4J' and '48.3%' fix rate are supported in Section 6.1.2 and Table 6/Figure 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims that retrieval 'improves' repair. The ablation study (RQ3) systematically isolates each component (BaseRepair, SigRepair, SnipRepair) and query rewriting (Table 5), providing adequate controlled single-variable manipulation for these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java benchmarks but makes broad claims: 'retrieving relevant code from the project codebase is essential for enhancing APR' (Section 9). The title 'Enhancing Automated Program Repair' does not specify Java. The abstract claims extend to 'APR tasks' generally."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (Section 7) is brief and does not discuss alternative explanations for the observed improvements — e.g., whether simply providing more context tokens (regardless of relevance) would help, or whether the improvement comes from increased sampling budget rather than retrieval quality."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures 'correct patches' (passing all tests and semantically equivalent to the fix) and claims to fix bugs. The measurement matches the claim granularity — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.3 says 'gpt-3.5 model from the ChatGPT family' without a specific version (e.g., gpt-3.5-turbo-0613). RQ2 uses 'ChatGPT-4', 'CodeLlama-13b', 'DeepSeek-33b' but without snapshot dates or API versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Only a fragment of the prompt is shown: 'You are an expert in program repair' (Section 4.1.1). The full prompts used for BaseRepair, SigRepair query rewriting (promptSIG), and SnipRepair are described in natural language but not provided in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 reports temperature=1. Retrieval parameters are specified: K=25 for SigRepair, K=15 intra + K=15 inter for SnipRepair. Patch counts: 1 (BaseRepair), 20 (SigRepair), 300 (SnipRepair). Timeout: 5 hours. α=0.5, β=0.5 initial weights."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage pipeline (BaseRepair → SigRepair → SnipRepair) is thoroughly described with formal algorithms (Algorithms 1-3), a framework diagram (Figure 3), and detailed descriptions of each stage's query rewriting, indexing, and retrieval components (Sections 4.1-4.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Sections 4.1.2 and 4.2.2 describe how retrieval datasets are constructed per bug: variable-based and file-based function selection for SigRepair, intra-file and inter-file function extraction for SnipRepair, with SentenceBERT and CodeBERT encoding. Section 5.1 describes the ManySStuBs4J sampling (30 per category)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' discusses internal threats (manual validation correctness) and external threats (representativeness of evaluation subjects)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 is brief (two short paragraphs) with fairly generic content. Internal threat: manual validation may be inaccurate (standard caveat). External threat: subjects may not be representative (standard caveat). No threats specific to this study's design choices are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No mention that results are limited to Java, to single-function bugs, to settings with perfect fault localization, or that they do not demonstrate practical applicability beyond benchmark settings."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Generated patches, retrieval results, and detailed per-bug outcomes are not released. Only aggregate counts appear in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 describes Defects4J V1.2 (391 bugs, 6 Java projects, single-function focus yielding 255 bugs) and ManySStuBs4J (150,000+ bugs, 16 categories, 30 per category selected for 480 total). Table 2 provides detailed statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (Defects4J V1.2, ManySStuBs4J)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "For ManySStuBs4J, the paper says 'we select 30 bugs from each category' (Section 5.1) without explaining the selection method (random sampling? first 30? specific criteria?). The path from raw benchmark data to final evaluation results is incompletely documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are clearly listed as affiliated with The University of Queensland, Brisbane, Australia. They do not evaluate a product from their own institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper uses OpenAI's API but there is no indication of OpenAI funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-3.5 (and gpt-4, CodeLlama, DeepSeek in RQ2) but never states training data cutoff dates for any model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J V1.2 (published 2014) and ManySStuBs4J (published 2020) are both well-known benchmarks that likely appear in GPT-3.5's training data. This potential overlap is never discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Both benchmarks were published years before GPT-3.5's training cutoff. The paper does not discuss contamination risk despite using models that could have memorized solutions from these widely-studied benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates automated tools on software benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows per-bug API costs (up to $0.27 for 300 patches in SnipRepair). Section 5.3 mentions a 5-hour timeout and 'average repair time is significantly lower, typically under 20 minutes.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-bug costs are shown in Figure 7, the total computational budget for the full experimental campaign (API spend across all 735+ bugs, total GPU time for embeddings) is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "LLM sampling uses temperature=1 (stochastic), but results are reported from what appears to be a single experimental run with no seed sensitivity analysis or multiple-seed results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 explicitly states the number of candidate patches per stage: 1 for BaseRepair, 20 for SigRepair, up to 300 for SnipRepair. The SigRepair process is repeated 20 times (Algorithm 2)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Figure 8 shows parameter sensitivity curves for K values in SigRepair and SnipRepair, but the total compute spent on search and the number of configurations tried across the full parameter space are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Figure 8 shows performance vs. parameter values with clear diminishing-returns justification for chosen K=25 (SigRepair) and K=15 (SnipRepair-Intra and SnipRepair-Inter). Selection is based on performance plateaus."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against multiple baselines across multiple projects and bug categories but performs no statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own system against baselines. For some baselines they 'reference reported outcomes from prior studies' (Section 5.2), but they do not acknowledge the systematic bias of authors evaluating their own method."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 directly plots the number of correct patches against API cost for both SigRepair and SnipRepair, showing the cost-performance tradeoff explicitly."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Defects4J V1.2 and ManySStuBs4J without discussing whether these benchmarks adequately measure real-world APR capability. No discussion of construct validity or comparison with alternative evaluation approaches."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "When comparing against ThinkRepair, RAP-Gen, and other baselines, different scaffolding/retrieval strategies are used. The paper does not disentangle whether improvements come from the retrieval mechanism vs. the additional context/sampling budget. RQ2 partially addresses this by testing multiple LLMs with the same scaffold."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J V1.2 (2014) and ManySStuBs4J (2020) both predate GPT-3.5's training data. Solutions to these bugs may be in the training data. This temporal leakage risk is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper assumes perfect fault localization (Section 6.1.1), providing statement-level fault information. While this follows prior work conventions, the paper does not discuss how this information leakage affects real-world applicability."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and test data are independent. The LLMs may have seen Defects4J bugs and their fixes during pre-training."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods (canary strings, membership inference, n-gram overlap analysis, decontamination) are used."
       }
     }
   },
   "claims": [
     {
       "claim": "RelRepair fixes 101 bugs in Defects4J V1.2, outperforming all compared LLM-based APR approaches.",
       "evidence": "Table 1 shows 101 total fixes vs. ThinkRepair (98), ChatRepair (76), Mulpor (70), RAP-Gen (42), and BaseChatGPT (47). Detailed per-project breakdown in Table 1 (Section 6.1.2).",
       "supported": "moderate"
     },
     {
       "claim": "RelRepair uniquely fixes 28 bugs not fixed by any other compared tool.",
       "evidence": "Venn diagram (Figure 4) shows 28 unique fixes when compared to RAP-Gen, Mulpor, and ThinkRepair on Defects4J V1.2 (Section 6.1.2). ChatRepair is excluded from the comparison.",
       "supported": "moderate"
     },
     {
       "claim": "RelRepair achieves a 48.3% fix rate on ManySStuBs4J, a 17.1% improvement over the 31.2% baseline.",
       "evidence": "Section 6.1.2 and Table 6/Figure 6 report fix rates per bug category and overall. Evaluation uses exact-match comparison against ground-truth fixes.",
       "supported": "moderate"
     },
     {
       "claim": "RelRepair with ChatGPT-4 fixes 120 bugs, a 37.2% improvement over base ChatGPT-4 (86 bugs).",
       "evidence": "Table 3 (Section 6.2.2) shows results across four LLMs with and without RelRepair on Defects4J V1.2.",
       "supported": "moderate"
     },
     {
       "claim": "Query rewriting is critical: removing it drops total fixes from 54 to 20.",
       "evidence": "Table 5 (Section 6.3.2) shows SigRepair drops from 15 to 4 fixes and SnipRepair drops from 39 to 16 fixes without query rewriting.",
       "supported": "moderate"
     },
     {
       "claim": "SnipRepair is more effective than SigRepair for complex bugs.",
       "evidence": "Table 4 shows SnipRepair fixes 39 bugs vs. SigRepair's 15 on Defects4J V1.2. Section 6.3.2 analyzes per-category differences on ManySStuBs4J showing SnipRepair excels on categories requiring deeper understanding.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "All comparisons are based on raw counts without any statistical tests, confidence intervals, or error bars. The difference between RelRepair (101) and ThinkRepair (98) is 3 bugs — without statistical testing, this difference may not be meaningful, especially with stochastic LLM sampling at temperature=1."
     },
     {
       "flag": "Benchmark contamination risk ignored",
       "detail": "Defects4J V1.2 (2014) and ManySStuBs4J (2020) are both well-known benchmarks likely present in GPT-3.5/4 training data. The paper never discusses whether the LLMs may have memorized solutions, which could inflate results for all approaches but especially the baseline."
     },
     {
       "flag": "Perfect fault localization assumption",
       "detail": "All experiments assume perfect statement-level fault localization (Section 6.1.1). This significantly simplifies the problem and inflates results compared to real-world settings where fault localization is itself a hard problem."
     },
     {
       "flag": "ManySStuBs4J exact-match evaluation",
       "detail": "For ManySStuBs4J, 'correctness of generated fixes is assessed solely by directly comparing the generated fix to the provided ground-truth solution; only exact matches are considered correct' (Section 6.1.1). This may severely undercount functionally correct but syntactically different patches."
     },
     {
       "flag": "Selective baseline comparison",
       "detail": "ChatRepair is excluded from the Venn diagram comparison (Figure 4) due to 'unavailability of its generated patches on GitHub.' This omits a key baseline from the uniqueness analysis while still including its numbers in Table 1."
     },
     {
       "flag": "No code released for a tool paper",
       "detail": "Despite proposing a concrete tool/system with algorithms and implementation details, no source code repository is provided, preventing independent verification or reproduction."
     }
   ],
   "cited_papers": [
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "LLM-based APR using ChatGPT's conversational interface for iterative repair, direct baseline in this evaluation."
     },
     {
       "title": "ThinkRepair: Self-directed automated program repair",
       "authors": ["X. Yin", "C. Ni", "S. Wang", "Z. Li", "L. Zeng", "X. Yang"],
       "year": 2024,
       "relevance": "Chain-of-Thought-based APR technique, strongest baseline achieving 98 fixes on Defects4J V1.2."
     },
     {
       "title": "RAP-Gen: Retrieval-augmented patch generation with CodeT5 for automatic program repair",
       "authors": ["W. Wang", "Y. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2023,
       "relevance": "Retrieval-augmented APR using historical bug-fix pairs, closest related approach to RelRepair."
     },
     {
       "title": "One size does not fit all: Multi-granularity patch generation for better automated program repair",
       "authors": ["B. Lin", "S. Wang", "M. Wen", "L. Chen", "X. Mao"],
       "year": 2024,
       "relevance": "Multi-granularity LLM-based APR approach, baseline in Defects4J comparison."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Foundational study on LLM-based APR establishing experimental conventions followed by this paper."
     },
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2404.05427",
       "relevance": "Autonomous agentic APR integrating AST analysis, represents agentic approach to program repair."
     },
     {
       "title": "Retrieval-augmented generation for knowledge-intensive NLP tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper that RelRepair's retrieval strategy is based on."
     },
     {
       "title": "CURE: Code-aware neural machine translation for automatic program repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "Code-aware NMT approach to APR, representative of learning-based APR methods."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM used as one of four models in RelRepair's multi-LLM evaluation (RQ2)."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["R. Just", "D. Jalali", "M. D. Ernst"],
       "year": 2014,
       "relevance": "Primary evaluation benchmark for APR research, used by RelRepair and all baselines."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM APR establishing experimental conventions followed in this paper."
     },
     {
       "title": "How often do single-statement bugs occur? The ManySStuBs4J dataset",
       "authors": ["R.-M. Karampatsis", "C. Sutton"],
       "year": 2020,
       "relevance": "Large-scale single-statement bug dataset used as second evaluation benchmark in this paper."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RAG for APR is a practically useful technique that practitioners could adapt, but no code is released for direct use."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirming that providing relevant context helps LLMs generate better code patches is expected and aligns with conventional wisdom about RAG."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on bug fixing, not adversarial or harmful applications."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy, no challenges to existing benchmarks or claims by others."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository, demo, or installable tool is released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses ChatGPT/GPT-4 which are well-known, but the authors are from an academic lab without high brand recognition."
     }
   }
 }