ai-research-survey

scan.json (32267B)

---

 {
   "paper": {
     "title": "Reinforcement Learning for Mutation Operator Selection in Automated Program Repair",
     "authors": [
       "Carol Hanna",
       "Aymeric Blot",
       "Justyna Petke"
     ],
     "year": 2023,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2306.05792",
     "doi": "10.1007/s10515-025-00501-z"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "The paper evaluates RL-based mutation operator selection for heuristic-based automated program repair across 30,080 independent repair attempts on 353 Defects4J bugs. Epsilon-greedy with average credit assignment was the most effective RL strategy, but did not significantly improve the number of bugs patched compared to baseline random selection. The RL approach generated more test-passing variants, suggesting the fitness function is too coarse (Boolean pass/fail) to effectively guide learning. Grouping 18 PAR templates into 7 semantically meaningful arms improved performance over using all 18 individually.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section XI states 'All source code, supplementary materials, and instructions needed to replicate our results are publicly available at: https://anonymous.4open.science/r/mutationLearner.' A URL to a code repository is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available Defects4J 2.0.0 benchmark and references the publicly available JaRFly replication package (ref [48]). The artifact link also contains supplementary materials."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section V-C describes hardware specifications of the compute cluster (RAM, SSD, CPU ranges) but does not provide software dependency specifications such as requirements.txt, Dockerfile, or specific library versions needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section XI provides a link to the artifact with 'instructions needed to replicate our results.' Section V-C describes search parameters, hyperparameters (Table II), and references JaRFly's replication scripts used for launching experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (III-IX) report only point estimates for success rates, bug counts, and variant numbers. No confidence intervals, error bars, or ± notation are reported anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., 'epsilon-greedy algorithm with average credit assignment is the best reinforcement learning-based mutation selection strategy') based solely on comparing raw percentages and counts. No statistical significance tests (p-values, t-tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as percentage differences with baseline context throughout. For example, Section VI-C: 'epsilon-greedy experiment 43 unique bugs were patched (2 more than the baseline) with a 4% higher success rate.' Tables provide both absolute numbers and rates enabling comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B justifies using 353 bugs (same set as JaRFly for direct comparison). Section V-C justifies 20 repeat runs 'to account for the heuristic nature of the underlying genetic algorithm.' They justify not extending to 835 bugs due to computational cost ('about 3.5 years of continuous computation')."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "While 20 independent repair attempts are conducted per bug, only aggregate success rates and median/average variant numbers are reported. No standard deviations, interquartile ranges, or other spread measures across the 20 runs are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is JaRFly's default uniform random selection strategy. Table V presents baseline results with both GenProg (3 operators) and GenProg+PAR (18 operators). All RL variants are compared against this baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baseline is the standard uniform selection approach used in JaRFly, which is described as a 'novel open-source framework for search-based APR' and represents the current standard practice in heuristic-based APR. The choice is justified in Section V-A."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies individual components: 4 selection strategies (PM, AP, epsilon-greedy, UCB), 2 credit assignment techniques, 2 reward types (raw vs relative fitness), 2 activation points, and 3 arm configurations (3, 7, 18). Table VII presents a comprehensive ablation of these factors."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported throughout: success rate (% of repair attempts producing patches), unique bugs patched, average variant number, median variant number, and patch quality scores (min, mean, median, max, % at 100% quality in Table IX)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-E states 'after manual investigation of the patches, we found that only 2 of the 93 patches in the baseline were correct and corresponded to 1 unique bug. None of the patches generated using the RL strategy were correct.' Manual evaluation of patch correctness was performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C describes using held-out evaluation test suites: 'The authors of JaRFly use two versions of the automated test generation tool EvoSuite to create held-out evaluation test suites.' These are separate from the test oracle used during repair."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by operator selection strategy (Tables IV, VI), credit assignment technique (Table IV), number of arms (Tables V, VII), reward type (Table VIII), and activation point (Table VII). Table I breaks down bugs by Defects4J project."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses failures: the 'every mutation' activation point yielded 0% success rate (Table VII), RL with 18 arms didn't improve over baseline (Section VI-D), and none of the RL patches on the 289 previously-unpatched bugs were correct (Section VI-E). Section VII discusses three hypothesized reasons for failure."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper is fundamentally a negative result: RL-based selection did not improve the number of bugs patched over random baseline. They report that every-mutation activation yielded 0% success, 40-generation experiments performed worse, and none of the RL patches on new bugs were correct."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims RL 'results in a higher number of test-passing variants, but does not exhibit a noticeable improvement in the number of bugs patched.' This is directly supported by results in Tables IV-IX. The abstract appropriately hedges with 'has not shown such improvements.'"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims such as 'epsilon-greedy is the most effective mutation operator selection strategy' are supported by controlled experiments where only the selection strategy varies while all other parameters are held constant (same benchmark, same tool, same search settings, same 20 repeated runs)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states evaluation is 'on 353 real-world bugs from the Defects4J benchmark.' Section IX explicitly acknowledges 'our methodology uses Java which may not generalise to other programming languages' and discusses external validity threats specific to their benchmark and language choices."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses three alternative explanations for why RL didn't improve APR: (1) learning too slow within the budget, (2) fitness function too coarse (Boolean) to guide learning effectively, (3) edit type may not be sufficient information to steer toward correct patches."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures bugs patched, success rate, and variant efficiency, which directly correspond to the claimed outcomes. They additionally assess patch quality using held-out test suites (Table IX), distinguishing between test-suite-adequate patches and correct patches via manual evaluation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies Defects4J version 2.0.0 and EvoSuite versions (v1.0.3 and v1.0.6), but describes JaRFly only as 'the latest version' without providing a specific version number or commit hash. JaRFly is the central tool and its exact version matters for reproducibility."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use any prompting or LLMs. The approach implements RL algorithms from scratch for mutation operator selection in genetic programming."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table II lists all hyperparameters: Pmin, Pmax, beta (0.8), epsilon (0.2), E (10). Section V-C reports search parameters: 10 generations, population size 40, 20 repair attempts. Section VI-A reports tuned learning rates for each strategy. Preliminary experiments test alpha values of 0.2, 0.4, 0.6, 0.8."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach modifies the mutation operator selection within JaRFly's genetic programming search."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B documents the data pipeline: started with 395 bugs from JaRFly, excluded Mockito project, removed 4 deprecated bugs, resulting in 353 bugs. Table I provides per-project breakdown. The preliminary subset selection (5 bugs, first lexicographically from each project) is also documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section IX 'Threats to Validity' is a dedicated section covering external, internal, and construct validity threats with substantive discussion across multiple paragraphs."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section IX identifies specific threats: Defects4J generalizability, Java language limitation, heterogeneous cluster hardware, JaRFly baseline reproduction discrepancy (41/49 vs 49 reported), exclusion of 3 PAR templates, and version mismatch issues."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states 'our methodology uses Java which may not generalise to other programming languages' and 'Our choices of benchmark and language enable direct comparison with state-of-the-art approaches.' They explicitly note the approach was tested only on Defects4J and with one APR tool."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section XI provides a link to 'All source code, supplementary materials, and instructions needed to replicate our results' at the anonymous artifact repository. The JaRFly replication package (ref [48]) with bug details and repair scripts is also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B describes data collection: used Defects4J bugs from the JaRFly study, explains selection criteria (same bugs as JaRFly, excluding Mockito and deprecated bugs). Table I provides the complete breakdown by project with counts of active and patched bugs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public benchmark (Defects4J), a well-documented collection of real-world Java bugs."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: 395 JaRFly bugs → exclude Mockito → remove 4 deprecated → 353 bugs (Table I). For RQ1-2: 49 GenProg-patched bugs. For RQ3: 64 GenProg+PAR bugs. For RQ4: 353 total bugs. Preliminary tuning used 5 bugs (first lexicographically per project)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text. The research appears to be academic (UCL and Université de Rennes affiliations) but funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University College London and Université de Rennes. These are academic institutions with no obvious conflict regarding the tools or benchmarks evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, independence cannot be assessed. Academic affiliations suggest independence, but absence of funding disclosure means this criterion is not met."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. The RL algorithms learn online from scratch during each repair attempt. There is no pre-trained model with a training data cutoff to report."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is evaluated. The RL approach learns operator selection probabilities from scratch during the repair process for each bug. Train/test overlap in the pre-trained model sense does not apply."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained model is used. The RL algorithms are implemented from scratch and learn during each repair session. Benchmark contamination through pre-training data is not applicable."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study is a benchmark evaluation of RL-based mutation operator selection on Defects4J software bugs."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study analyzes automated program repair on software bugs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VI reports 'Computation time for each repair attempt ranged, on average, from 45 minutes to 1.5 hours, up to 12 hours in longest runs.' The total budget is described as '30,080 independent repair attempts' amounting to 'about 3.5 years of continuous computation.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C describes the compute cluster (1028 nodes with varying specs). Section VI states the total of 30,080 repair attempts amounting to about 3.5 years of continuous computation or 'multiple months of active cluster usage.' Hardware details include RAM (16-375GB), SSD (80-780GB), Intel Xeon CPUs (4-48 cores)."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While 20 independent repair attempts are run per bug, results are only reported as aggregate success rates and median/average variant numbers. Section VI-C notes 'We observed variance even when the same seeds were used' but no systematic seed sensitivity analysis is presented."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C clearly states 'For each bug, 20 repair attempts were launched independently, to account for the heuristic nature of the underlying genetic algorithm.' The preliminary study used 1600 total runs (Section V-C). Total: 30,080 repair attempts (Section VI)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C describes preliminary experiments: 4 learning rates × 4 operator selection strategies × 5 bugs × 20 repeats = 1600 repair runs for tuning. The 4 alpha values (0.2, 0.4, 0.6, 0.8) and selection criteria are clearly stated in Table III."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A states 'For each operator selection strategy, we chose the learning rate value that maximised the number of successful repair attempts.' Selection was on a separate subset of 5 bugs (~10% of the 49 patched bugs), preventing overfitting to the full evaluation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 4 selection strategies × 2 credit assignments × 2 reward types × multiple arm configurations without applying any correction for multiple comparisons. No statistical significance tests are performed at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement RL variants within JaRFly and compare against JaRFly's own baseline. While they note baseline replication discrepancies (41/49 vs 49 in Section VI-C), they do not explicitly discuss the bias inherent in evaluating their own system against their own baseline implementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "All main experiments use matched compute budgets (10 generations × 40 population = 400 evaluations per attempt). The 40-generation variant (40 × 10 = 400) maintains the same total evaluations. Table VII explicitly compares different generation/population configurations at matched total compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper justifies using Defects4J as 'the most comprehensive and popular dataset for evaluating Java APR tools' (Section V-B) but does not discuss whether it adequately measures the capabilities the paper claims to evaluate, or whether the benchmark has known limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach modifies mutation operator selection within a single tool (JaRFly) with all other components held constant."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the time between bug creation and the experiment could affect results, or whether the repair tool could benefit from temporal information leakage through the code corpus used for mutations."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the test oracle provides information that could constitute feature leakage, or whether the fitness function based on test passing leaks information about the correct fix."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential non-independence among Defects4J bugs (e.g., bugs from the same project sharing structural properties) or how this could affect generalizability of results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The paper does not use temporal splits, decontamination, or any other leakage mitigation technique."
       }
     }
   },
   "claims": [
     {
       "claim": "Average credit assignment is best suited for mutation operator selection in heuristic-based APR, indicating the search process is stationary.",
       "evidence": "Table IV shows that in all experiments (PM, UCB, epsilon-greedy), the success rate with average credit assignment is higher than with exponential recency-weighted average. Only AP shows a marginal advantage for exponential on unique bugs patched (Section VI-B).",
       "supported": "moderate"
     },
     {
       "claim": "Epsilon-greedy is the most effective mutation operator selection strategy, while probability matching is the most efficient.",
       "evidence": "Table IV shows epsilon-greedy achieves 47.9% success rate and patches 43/49 bugs (highest). Table VI shows PM has the lowest median (50) and average (97) variant numbers among RL strategies for common bugs (Section VI-C).",
       "supported": "moderate"
     },
     {
       "claim": "Increasing the number of mutation operators from 3 to 18 does not improve bug patching with RL-based selection.",
       "evidence": "Section VI-D reports 18-arm epsilon-greedy achieved 35.2% success rate and 51 unique bugs vs baseline 35.6% and 52 bugs with 18 operators (Table V).",
       "supported": "strong"
     },
     {
       "claim": "Grouping PAR templates into 7 semantic arms improves over 18 individual arms.",
       "evidence": "Table VII shows 7 arms achieve 43% success rate vs 31% for 18 arms on the 5-bug sample. Table VIII shows 7-arm configuration patches 48-51 bugs on the 64-bug set.",
       "supported": "moderate"
     },
     {
       "claim": "RL-based mutation operator selection generates more test-passing variants but does not significantly improve bugs patched compared to baseline.",
       "evidence": "Tables IV-VIII show RL variants have comparable or slightly better success rates but similar unique bug counts. Section VI-E shows none of the RL patches on the 289 previously-unpatched bugs were correct vs 2 correct baseline patches.",
       "supported": "strong"
     },
     {
       "claim": "The coarse Boolean fitness function limits the effectiveness of RL-guided operator selection.",
       "evidence": "Section VII hypothesizes this based on observed results. Table VII shows 'every mutation' activation (more frequent updates) yields 0% success, suggesting the RL signal is insufficient. No direct experimental test of this hypothesis is provided.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (e.g., 'epsilon-greedy is the most effective') are based on comparing raw percentages and counts without any statistical significance tests. With 20 runs per bug, the observed differences (e.g., 43.9% vs 47.9% success rate) could be within normal variation."
     },
     {
       "flag": "Baseline reproduction failure",
       "detail": "The authors could only reproduce 41/49 bugs from the JaRFly baseline despite using the same parameters and seeds. This 16% discrepancy casts doubt on the comparability of the baseline and experimental results, attributed to version differences and excluded PAR templates."
     },
     {
       "flag": "Preliminary tuning on only 5 bugs",
       "detail": "Hyperparameter tuning (learning rate selection) was performed on only 5 bugs, which is ~10% of the 49 bugs used for RQ1-2 evaluation. This tiny tuning set may not be representative, and the first-lexicographically selection criterion is arbitrary."
     },
     {
       "flag": "No variance or uncertainty measures",
       "detail": "Despite running 20 independent trials per bug, the paper never reports standard deviations, confidence intervals, or any measure of result stability across runs. This makes it impossible to assess whether observed differences are meaningful."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated program repair",
       "authors": ["C. Le Goues", "M. Pradel", "A. Roychoudhury"],
       "year": 2019,
       "relevance": "Comprehensive survey of APR approaches spanning constraint-based, learning-based, and heuristic-based techniques."
     },
     {
       "title": "GenProg: A generic method for automatic software repair",
       "authors": ["C. Le Goues", "T. V. Nguyen", "S. Forrest", "W. Weimer"],
       "year": 2012,
       "relevance": "Foundational heuristic-based APR tool using genetic programming with statement-level mutations."
     },
     {
       "title": "Sapfix: Automated end-to-end repair at scale",
       "authors": ["A. Marginean", "J. Bader", "S. Chandra", "M. Harman", "Y. Jia", "K. Mao", "A. Mols", "A. Scott"],
       "year": 2019,
       "relevance": "Industrial-scale automated program repair deployment at Meta, demonstrating real-world APR adoption."
     },
     {
       "title": "ARJA: Automated Repair of Java Programs via Multi-Objective Genetic Programming",
       "authors": ["Y. Yuan", "W. Banzhaf"],
       "year": 2018,
       "relevance": "Multi-objective genetic programming approach to APR evaluated on Defects4J benchmark."
     },
     {
       "title": "Quality of Automated Program Repair on Real-World Defects",
       "authors": ["M. Motwani", "M. Soto", "Y. Brun", "R. Just", "C. Le Goues"],
       "year": 2022,
       "relevance": "JaRFly framework used as the base tool in this study, with patch quality evaluation methodology on Defects4J."
     },
     {
       "title": "Automatic Patch Generation Learned from Human-Written Patches",
       "authors": ["D. Kim", "J. Nam", "J. Song", "S. Kim"],
       "year": 2013,
       "relevance": "Introduces PAR templates for APR, providing 18 fix templates used as mutation operators in this study."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of chatgpt",
       "authors": ["D. Sobania", "M. Briesch", "C. Hanna", "J. Petke"],
       "year": 2023,
       "relevance": "Evaluates LLM-based (ChatGPT) automated program repair performance on benchmarks."
     },
     {
       "title": "Cure: Code-aware neural machine translation for automatic program repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "Neural machine translation approach to APR demonstrating learning-based repair techniques."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM-based APR approach, demonstrating the shift toward language model-based repair."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Evaluates large pre-trained language models for automated program repair."
     },
     {
       "title": "On the introduction of automatic program repair in bloomberg",
       "authors": ["S. Kirbas", "E. Windels", "O. Mcbello"],
       "year": 2020,
       "relevance": "Industrial deployment of heuristic-based APR at Bloomberg, demonstrating real-world adoption."
     },
     {
       "title": "Is the cure worse than the disease? Overfitting in automated program repair",
       "authors": ["E. K. Smith", "E. T. Barr", "C. Le Goues", "Y. Brun"],
       "year": 2015,
       "relevance": "Identifies the overfitting problem in APR where patches pass test suites but are incorrect."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The approach is implemented in JaRFly but shows no significant improvement over random selection, limiting immediate practical utility."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The negative result — that RL does not help APR despite helping other evolutionary algorithm domains — is mildly surprising but in a niche area."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised by this work on mutation operator selection."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper presents straightforward negative experimental results."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Source code is available but requires setting up JaRFly, Defects4J, and a Java environment — not easily demoed."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Academic work from UCL and Université de Rennes; not a well-known lab or product for general audiences."
     }
   }
 }