ai-research-survey

scan.json (32431B)

---

 {
   "paper": {
     "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
     "authors": [
       "Chunqiu Steven Xia",
       "Yuxiang Wei",
       "Lingming Zhang"
     ],
     "year": 2022,
     "venue": "arXiv",
     "arxiv_id": "2210.14179",
     "doi": "10.48550/arXiv.2210.14179"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This paper presents the first extensive evaluation of 9 pre-trained language models (125M to 20B parameters) for automated program repair across 5 benchmarks in 3 languages. PLMs directly applied without finetuning outperform all 20 existing APR tools on Defects4J 1.2, with 109 bugs correctly fixed vs 67 by the best baseline (AlphaRepair). The study demonstrates a consistent scaling effect where larger models fix more bugs with better compilation rates, and shows that infilling-style APR using suffix context significantly improves repair over generative-only approaches. Combining PLMs with simple repair templates and increased sampling (2000 samples) further boosts performance to 78 correct fixes.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section VI states 'we carefully performed the analysis and released the correct patches and code used to perform the experiments for public evaluation [74]' with a figshare link, though the URL (https://figshare.com/s/temp) appears to be a placeholder."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All 5 evaluation benchmarks (Defects4J 1.2/2.0, QuixBugs-Java/Python, ManyBugs) are publicly available standard APR datasets. Additionally, the authors claim to release patches via figshare [74]."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-B mentions 'Python using PyTorch' and specific hardware (32-Core workstation, Ryzen Threadripper PRO 3975WX, 256GB RAM, NVIDIA RTX A6000, Ubuntu 20.04.4 LTS), but no library versions, requirements.txt, or dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The figshare reference [74] claims to release code but the paper itself contains no reproduction guide or README description."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables III-V, VII-VIII, and X report only point estimates (number of correct/plausible patches). No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'substantially outperform all existing APR techniques') based solely on comparing raw counts without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides absolute and relative differences with context throughout, e.g., 'Codex can fix 32 more bugs than the existing best APR technique,' '40% (62/154) of the total bugs,' and percentage improvements for various comparisons."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes. The paper uses standard benchmarks (391/438/40/40/185 bugs) without discussing whether these sizes are sufficient for the claims made or performing any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or standard deviation is reported. Results are from a single experimental run with stochastic sampling (nucleus sampling with temperature 0.8), but no spread measures across repeated experiments are provided."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D describes comparison against 20 APR tools: 8 learning-based (AlphaRepair, RewardRepair, Recoder, DeepDebug, CURE, CoCoNuT, DLFix, SequenceR) and 12 traditional (TBar, PraPR, AVATAR, SimFix, FixMiner, CapGen, JAID, SketchFix, NOPOL, jGenProg, jMutRepair, jKali)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent state-of-the-art tools: AlphaRepair (2022), RewardRepair (2022), Recoder (2021), CURE (2021), which were all contemporary at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies repair settings (complete function, infilling, single line), compares generative vs infilling models on the same task (Table V), and RQ4 ablates sample size (200 vs 2000) and the addition of repair templates (Table X)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports correct patches, plausible patches, compilation rate (Figure 4), generation speed (Table VI), and entropy values (Table IX) as evaluation metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-E states 'we follow the standard practice in APR research and manually inspect each plausible patch for semantic equivalency' to determine correct patches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The PLMs are applied zero-shot (no finetuning on APR data). The evaluation benchmarks (Defects4J, QuixBugs, ManyBugs) serve as held-out test sets since no model tuning is performed on them."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (5 benchmarks), by repair setting (3 settings), by model (9 PLMs), and by error type (syntactic vs semantic in Figure 4). Tables III-V provide per-dataset, per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 analyzes syntactic and semantic error rates across models and settings. The paper discusses that generative models for single-line generation produce high syntactic error rates because they lack suffix context. Section VI discusses ManyBugs reproducibility failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: smaller models perform poorly (GPT-Neo 125M fixes very few bugs), generative models have high syntax error rates for single-line generation, complete function generation yields lower correct-to-plausible ratios than infilling, and generation speed drastically decreases with model size (71x slower for GPT-NeoX vs GPT-Neo 125M)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims PLMs 'substantially outperform all existing APR techniques on all our datasets' — supported by Tables VII-VIII showing 109 bugs fixed vs 67 best baseline on Defects4J 1.2. Scaling effect claim supported by Tables III-V. Suffix importance claim supported by Table V Codex comparisons."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'suffix code is important' are justified via controlled comparison of Codex with and without suffix on the same data (Table V). The scaling effect claim is supported by comparing multiple models of different sizes trained on the same data (GPT-Neo 125M/1.3B/2.7B, GPT-J, GPT-NeoX). RQ4 ablations use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to tested settings. The abstract qualifies 'on all our datasets.' Section VI-External states 'our findings may still not generalize to other datasets or languages.' Results are reported separately per dataset and language. The focus on single-function bugs under perfect fault localization is stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI extensively discusses data leakage as an alternative explanation, finding 15% of fixes appear in training data. They also discuss that Codex outperforms GPT-NeoX despite being smaller because it is 'designed and finetuned for code generation' (Section V-A1), offering an alternative to pure scaling."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures correct/plausible patches on specific benchmarks and frames results in those terms. It does not overclaim beyond 'number of bugs fixed' on the tested benchmarks. The distinction between plausible (test-passing) and correct (semantically equivalent to developer fix) patches is explicit."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I specifies model names with parameter counts (GPT-Neo 125M/1.3B/2.7B, GPT-J 6.7B, GPT-NeoX 20B, Codex 12B, CodeT5 220M, INCODER 1.3B/6.7B). Section IV-B specifies Codex uses 'code-davinci-002 engine.' Training datasets are identified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the complete prompt structure for function generation with a concrete Fibonacci example. Figures 2 and 3 show the infilling and single-line generation input formats. The prompt includes task description, examples, and the target bug."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B reports 'nucleus sampling with top p = 0.95, temperature = 0.8 and 200 samples per bug' and notes this is 'consistent with previous studies on PLMs [26], [28], [30].'"
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. PLMs are directly queried for patch generation without any scaffold, retry logic, or multi-step workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes filtering benchmarks to find bugs matching each repair setting, with Table II showing the number of single-function (#SF), single-hunk (#SH), and single-line (#SL) bugs per dataset. For ManyBugs, they state 'we only use the 91 bugs where the results were reproducible by us' out of 185."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides dedicated discussion of both internal and external threats, including data leakage analysis and generalizability concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses specific threats: manual validation of patches as an internal threat, detailed data leakage analysis finding 15% (20/128) of fixes appear in training data, and the specific limitation to 5 datasets across 3 languages."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope to single-function bugs (Section IV-C), evaluation under perfect fault localization (Section IV-D), 3 programming languages, and 5 specific benchmarks. Section VI states 'our findings may still not generalize to other datasets or languages.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only the correct patches and code are claimed to be released [74]. Raw generation outputs (all 200 samples per bug across all models) are not made available for independent verification of the filtering and validation process."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes each benchmark in detail: Defects4J from open-source Java projects (391/438 bugs), QuixBugs from programming challenge (40 bugs each in Java/Python), ManyBugs from C open-source projects (185 bugs, 91 reproducible)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (Defects4J, QuixBugs, ManyBugs)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark filtering (Table II showing bug counts per category) through prompt construction (Figures 1-3), generation (200 samples, nucleus sampling), filtering (removing syntactic/semantic errors per Figure 4), and test validation is documented. For ManyBugs, the reduction from 185 to 91 reproducible bugs is explained."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper text provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are listed with their affiliation at the University of Illinois at Urbana-Champaign. They evaluate third-party models and have no apparent affiliation with model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, making it impossible to assess funder independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff dates are stated for any of the models. Table I lists training datasets (The Pile, CodeSearchNet, BigQuery) but not the date ranges. Codex's training data is listed as 'N.R.' (not released)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI extensively discusses potential train/test overlap. The authors check whether fixed functions appear in training data for models with accessible training datasets, finding 15% (20/128) of developer-matching fixes are also found in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI addresses contamination for Defects4J 1.2 specifically, analyzing developer-patch matches (66% of fixes differ from developer patches), checking training data for overlap (15% found), and noting QuixBugs is unlikely in training data due to low GitHub stars and synthetic nature."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of PLMs on automated program repair datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates PLMs on publicly available benchmark datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation is automated using test suites and manual patch inspection by the authors."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Benchmark selection criteria are described in Section IV-C."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Manual patch inspection is not described as blinded."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table VI reports generation speed (patches/minute) for each locally-run model across datasets and settings. Section V-B1 states 'generating 200 patches for each of the 3 settings (i.e., at most 600 patches in total) costs no more than 2.5 hours for each model.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B specifies hardware (32-Core Ryzen Threadripper PRO 3975WX, 256GB RAM, NVIDIA RTX A6000, Ubuntu 20.04.4 LTS). Table VI provides generation speeds enabling total compute estimation. The 2.5-hour per-model budget is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Results are from a single experimental run with stochastic sampling, but no variance across seeds or repeated runs is provided."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B states '200 samples per bug' as the default, and RQ4 uses '2000 samples per bug.' The number of generated patches is explicitly stated for all experiments."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search was performed. The paper uses default settings from prior work (top_p=0.95, temperature=0.8) without exploring alternatives, despite acknowledging 'How to pick an optimal temperature value is not obvious for a problem such as APR.'"
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper justifies using default generation settings by stating 'This generation setting is consistent with previous studies on PLMs [26], [28], [30]' (Section IV-B). No cherry-picking of configurations is evident."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed in this paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors do not acknowledge the bias of evaluating their own system. They use baseline results 'directly from previous studies [17], [18], [24]' (Section IV-D), which partially mitigates re-implementation bias, but this advantage is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table VI reports generation speed for each model, and Section V-A2 explicitly discusses the 'trade-off between repair effectiveness and time cost when using large models' (e.g., GPT-NeoX is 71x slower than GPT-Neo 125M). The 2.5-hour budget is compared against baselines' 5-hour timeout."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses standard APR benchmarks (Defects4J, QuixBugs, ManyBugs) without questioning whether they adequately measure practical program repair capability. No discussion of construct validity or benchmark limitations beyond the synthetic nature of QuixBugs."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. PLMs are directly queried for patch generation, so there is no scaffold confound to address."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses that Defects4J 1.2 is 'the most widely studied dataset for APR' and checks whether patches appear in training data. They note QuixBugs 'is not part of the training data as it has low number of stars on GitHub and contains synthetic bugs.'"
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup (e.g., providing perfect fault localization, buggy function context) leaks information that would not be available in realistic repair scenarios. Perfect FL is used without framing it as a form of information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI checks whether fixed functions appear in the training datasets for models with accessible training data (GPT-Neo, GPT-J, GPT-NeoX, CodeT5), finding 15% (20/128) overlap. This directly addresses train/test non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The authors apply a concrete detection method: checking whether developer-patched functions appear in training datasets, finding 20/128 matches. They also compare PLM-generated patches against developer patches, finding 66% produce different fixes (234/354)."
       }
     }
   },
   "claims": [
     {
       "claim": "Directly applying PLMs can substantially outperform all existing APR techniques, fixing 109 bugs on Defects4J 1.2 vs 67 by the best baseline (AlphaRepair)",
       "evidence": "Table VII shows combined PLM results fix 109 single-function bugs. AlphaRepair fixes 67. Codex alone fixes 99. Figure 5 shows Venn diagram with 36 unique fixes by PLMs. (Section V-B1)",
       "supported": "strong"
     },
     {
       "claim": "There is a scaling effect for APR where larger PLMs fix more bugs",
       "evidence": "Tables III-V show consistent improvement from GPT-Neo 125M to 1.3B to 2.7B to GPT-J to GPT-NeoX across all 5 datasets and 3 repair settings. Figure 4 shows error rates decrease with model size. (Sections V-A1, V-A3)",
       "supported": "strong"
     },
     {
       "claim": "Suffix code (infilling-style) is important for generating more fixes with higher compilation rate",
       "evidence": "Table V compares Codex single-line (prefix only) vs Codex suffix (prefix+suffix), showing improvement across all datasets (e.g., 32→39 correct on Defects4J 1.2). Figure 4 shows infilling achieves lowest semantic error rates. (Sections V-A1, V-A3)",
       "supported": "strong"
     },
     {
       "claim": "PLMs consider correct patches to be more natural (lower entropy) than incorrect patches",
       "evidence": "Table IX shows mean entropy of correct patches is consistently lower than plausible patches, which are lower than non-plausible patches, across all models and settings. (Section V-C1)",
       "supported": "strong"
     },
     {
       "claim": "Entropy-based patch ranking enables faster validation of correct patches compared to random ordering",
       "evidence": "Figure 7 shows entropy rankings (both mean and sum) consistently outperform random patch ordering on Defects4J 1.2 across 5 PLMs. (Section V-C2)",
       "supported": "moderate"
     },
     {
       "claim": "PLM-based APR can be further boosted by increasing sample size and incorporating repair templates",
       "evidence": "Table X shows INCODER goes from 37 (200 samples) to 64 (2000 samples) to 78 (with templates) correct fixes on Defects4J 1.2, surpassing all baselines. (Section V-D)",
       "supported": "strong"
     },
     {
       "claim": "The majority of correct PLM fixes are not from memorizing training data",
       "evidence": "Section VI reports 66% of fixes differ from developer patches. Only 15% (20/128) of developer-matching fixes appear in accessible training data. QuixBugs results are strong despite likely not being in training data. (Section VI)",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('substantially outperform', 'scaling effect') are based on raw count comparisons without any statistical tests, despite stochastic sampling that could produce different results across runs."
     },
     {
       "flag": "No variance across runs reported",
       "detail": "Results use nucleus sampling (temperature=0.8) which is stochastic, yet only single-run results are reported. The number of correct fixes could vary across runs, but no variance or confidence intervals are provided."
     },
     {
       "flag": "Data leakage for best-performing model",
       "detail": "Codex produces the strongest results but its training data is 'N.R.' (not released), making it impossible to verify contamination. The authors can only check leakage for open-source models, which are not the top performers."
     },
     {
       "flag": "Perfect fault localization assumption",
       "detail": "All main comparisons use perfect fault localization (ground-truth bug location known), which is unrealistic for practical deployment. The paper's title emphasizes 'Practical Program Repair' but this setting is not practical."
     },
     {
       "flag": "Artifact URL appears to be placeholder",
       "detail": "Reference [74] for released code and patches points to 'https://figshare.com/s/temp' — the 'temp' suggests this URL was never replaced with the actual figshare link."
     }
   ],
   "cited_papers": [
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2207.08281",
       "relevance": "Proposes AlphaRepair, the zero-shot PLM-based APR approach using CodeBERT that this paper extends and substantially outperforms."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex, the best-performing PLM in this study, foundational work on LLMs for code generation."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo", "Duyu Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Pre-trained code model used by AlphaRepair, demonstrating feasibility of pre-trained models for code understanding tasks."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Code infilling model evaluated in this study, demonstrating strong APR results using causal masking for bidirectional generation."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom B. Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Introduces GPT-3 and few-shot learning paradigm that underlies the zero-shot/few-shot PLM application strategy for APR."
     },
     {
       "title": "Patch generation with language models: Feasibility and scaling behavior",
       "authors": ["Samuel D. Kolak", "Ruben Martins", "Claire Le Goues", "Vincent J. Hellendoorn"],
       "year": 2022,
       "relevance": "Prior work evaluating Codex and smaller PLMs for patch generation, demonstrating scaling behavior on a small synthetic dataset."
     },
     {
       "title": "Can OpenAI's Codex fix bugs? An evaluation on QuixBugs",
       "authors": ["Julian A. Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "relevance": "Prior evaluation of Codex for bug fixing on QuixBugs, one of the first studies applying commercial PLMs to APR."
     },
     {
       "title": "A syntax-guided edit decoder for neural program repair (Recoder)",
       "authors": ["Qihao Zhu", "Zeyu Sun", "Yuan-an Xiao"],
       "year": 2021,
       "relevance": "State-of-the-art learning-based APR tool using syntax-guided generation, one of the main baselines outperformed by PLMs."
     },
     {
       "title": "CURE: Code-aware neural machine translation for automatic program repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "Learning-based APR tool using code-aware NMT, a key baseline demonstrating limits of supervised APR approaches."
     },
     {
       "title": "Neural program repair with execution-based backpropagation (RewardRepair)",
       "authors": ["He Ye", "Matias Martinez", "Martin Monperrus"],
       "year": 2022,
       "relevance": "Learning-based APR tool using execution feedback, represents the state-of-the-art in supervised program repair."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["René Just", "Darioush Jalali", "Michael D. Ernst"],
       "year": 2014,
       "relevance": "The most widely-used APR benchmark, central to the evaluation in this paper and most APR research."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Establishes scaling laws for language models, providing theoretical basis for the scaling effect observed in this paper's APR results."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly actionable — practitioners can use off-the-shelf PLMs via HuggingFace or Codex API for automated bug fixing with the described prompt strategies."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Shows general-purpose PLMs without any APR-specific training substantially outperform all 20 dedicated APR tools, challenging the value of domain-specific APR engineering."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or security concerns raised; the paper focuses on beneficial application of PLMs for fixing software bugs."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild tension: years of specialized APR tool development are outperformed by generic language models used zero-shot, potentially undermining the APR research agenda."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Uses publicly available models from HuggingFace (GPT-Neo, INCODER) and Codex API, though no standalone tool or demo is released."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Features OpenAI's Codex prominently as the best performer; GPT-Neo/J/NeoX and INCODER are well-known in the ML community."
     }
   }
 }