ai-research-survey

scan.json (30143B)

---

 {
   "paper": {
     "title": "GAMMA: Revisiting Template-based Automated Program Repair via Mask Prediction",
     "authors": [
       "Quanjun Zhang",
       "Chunrong Fang",
       "Tongke Zhang",
       "Bowen Yu",
       "Weisong Sun",
       "Zhenyu Chen"
     ],
     "year": 2023,
     "venue": "International Conference on Automated Software Engineering (ASE)",
     "arxiv_id": "2309.09308",
     "doi": "10.1109/ASE56229.2023.00063"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "GAMMA combines template-based APR fix patterns with pre-trained language model mask prediction to generate donor code, fixing 82 bugs on Defects4J-v1.2 (81.19% precision), outperforming all compared APR techniques. The approach generalizes to Defects4J-v2.0 (45 bugs) and QuixBugs (22 bugs), and is scalable across different pre-trained models (UniXcoder: 82, CodeBERT: 80, ChatGPT: 67). Data leakage investigation found only 3 overlapping bugs in the pre-training data, with GAMMA still outperforming baselines when these are excluded.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The authors state 'we release the relevant materials (including source code, experimental results, and correct patches) in our experiment for replication and future research' with a GitHub link (reference [29]: https://github.com/iSEngLab/GAMMA)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (Defects4J-v1.2, Defects4J-v2.0, QuixBugs) are publicly available standard benchmarks. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Ubuntu 18.04.3 server with two Tesla V100-SXM2 GPUs' and 'unixcoder-base' model, but provides no software dependency versions (Python, PyTorch, etc.), requirements.txt, or Dockerfile."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the approach at a conceptual level but does not include step-by-step reproduction instructions, specific commands to run, or a detailed README. A GitHub repository is referenced but the paper itself lacks reproduction guidance."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '82 bugs', '45 bugs') with no confidence intervals, error bars, or uncertainty measures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GAMMA 'substantially outperforms' baselines based solely on comparing raw bug counts (e.g., 82 vs 68 for TBar). No statistical significance tests are applied to any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context: '20.59% (14 bugs) and 26.15% (17 bugs) improvement over TBar and Recoder' and precision rates '81.19% (82/101) for plausible patches, 9.61% higher than TBar (68/95)'."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the benchmark sizes (395 bugs in Defects4J-v1.2, 257 in v2.0, 40 in QuixBugs). No power analysis or discussion of whether these sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are single-run numbers with no standard deviation, variance, or spread measures reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "GAMMA is compared against 9 APR tools: SequenceR, CoCoNuT, CURE, DLFix, Recoder, AlphaRepair, CIRCLE, PraPR, and TBar (Table I)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent state-of-the-art tools: AlphaRepair (ESEC/FSE 2022), CIRCLE (ISSTA 2022), Recoder (ESEC/FSE 2021), all published within 1-2 years of this work."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "GAMMA has multiple components (fix template transformation, context construction with comment line, mask prediction model) but no systematic ablation study isolates their individual contributions. RQ3 varies the pre-trained model but does not ablate other components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: number of plausible patches, number of correct patches, and precision (correct/plausible ratio). Table I shows both plausible and correct counts (e.g., '82/101')."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three authors manually verify all plausible patches: 'A plausible patch is considered to be correct if all three authors identify it as equivalent to a ground truth patch semantically' (Section VI)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "GAMMA uses UniXcoder in a zero-shot setting with no fine-tuning on any bug-fixing data. Defects4J and QuixBugs serve as independent test sets that were never used for any model selection or tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides per-project breakdowns across all 6 Defects4J-v1.2 projects (Chart, Closure, Lang, Math, Mockito, Time) for all compared techniques."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows case studies where GAMMA succeeds but TBar fails (Listings 3, 4). However, no specific examples or analysis of where GAMMA itself fails are provided. Template coverage limitations are mentioned only in passing."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "ChatGPT-based GAMMA fixes only 67 bugs compared to UniXcoder's 82 (Section V-C). GAMMA also underperforms AlphaRepair on QuixBugs (22 vs 28, Table II). Template limitations for new benchmarks are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are supported: 82 bugs on Defects4J-v1.2 (Table I), 20.59% and 26.15% improvements over TBar and Recoder (Table I), 45 bugs on Defects4J-v2.0 and 22 on QuixBugs (Table II), CodeBERT-based (80) and ChatGPT-based (67) variants (Section V-C)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim is that mask prediction improves donor code generation over local file search. The comparison with TBar (same templates, different donor code strategy) provides a controlled single-variable manipulation. RQ3 further validates by varying only the pre-trained model component."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Automated Program Repair via Mask Prediction' makes no language restriction, but all evaluation is on Java programs only. The abstract and contributions section make broad claims about APR without bounding to Java or to the specific template types used."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI (Threats to Validity) discusses three specific alternative explanations: manual inspection bias, fault localization settings biasing results, and potential data leakage in pre-trained models affecting observed performance."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures number of correctly-fixed bugs verified by manual inspection against developer patches, which is a direct measure of repair capability. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "UniXcoder is specified as 'unixcoder-base' (Section IV-E), and ChatGPT is specified as 'gpt-3.5-turbo-0301, which is the latest version available' (Section V-C). CodeBERT is referred to simply as 'CodeBERT' without a specific checkpoint identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The ChatGPT prompt is provided verbatim: 'Next token prediction task, the first line is a comment to help prediction, just return 250 possible predictions for <mask> with highest probability:' (Section V-C). UniXcoder input construction (comment line + method with masked tokens) is described in detail (Section III-C)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Beam size is set to 250 (Section IV-E), mask number range 1-20 for CodeBERT (Section V-C), and 5-hour running-time limit per bug. The prediction mode (beam search for UniXcoder, sequential for CodeBERT) is described."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "GAMMA does not use agentic scaffolding. It is a pipeline of template selection → mask prediction → patch validation with no tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) AST parsing with Eclipse JDT for template selection (Section III-B), (2) input construction with comment line + method context (Section III-C), (3) mask prediction and candidate patch generation, (4) compilation filtering and test suite validation (Section III-D)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI 'Threats to Validity' provides substantive discussion of three specific threats: manual inspection bias, fault localization settings, and data leakage of pre-trained models."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: (1) mitigating manual inspection bias with three independent reviewers, (2) acknowledging perfect fault localization may not reflect practice, (3) querying pre-training datasets to find 3 leaked bugs (Closure-73, Closure-126, Time-19) and verifying results hold without them."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: fix templates are 'designed for Java' (Section V-B), 'most of the templates are summarized from Defects4J-v1.2, which may mean that some templates cannot be applied to any bugs except those from Defects4J-v1.2' (Section V-B), and perfect fault localization 'may bring bias in repair performance' (Section VI)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The authors release 'source code, experimental results, and correct patches' via GitHub (reference [29]), enabling independent verification of the main results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data sources are well-described: Defects4J-v1.2 (395 bugs from 6 open-source Java projects), Defects4J-v2.0 (257 single-location bugs from 17 projects), QuixBugs (40 Java programs). Each benchmark's composition and origin are cited."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard public benchmarks (Defects4J, QuixBugs)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: buggy program → fault localization → AST-based template selection → mask token insertion → UniXcoder beam search prediction → compilation filtering → test suite validation → manual inspection. Each step is described in Sections III-A through III-D."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgment section: 'This work is supported partially by the National Natural Science Foundation of China (61932012, 62141215).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from Nanjing University's State Key Laboratory for Novel Software Technology. They are not evaluating a product from their own institution or company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The National Natural Science Foundation of China is a government funding body with no commercial interest in GAMMA's repair performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest declaration is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff date is stated for UniXcoder, CodeBERT, or ChatGPT. The paper mentions UniXcoder was pre-trained on CodeSearchNet data (2.3M NL-PL pairs and 4.1M unimodal code) but does not state when this data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI explicitly addresses overlap: 'we query the pre-training datasets including 2.3M functions paired with comments and 4.1M unimodal code from CodeSearchNet' and found 3 bugs leaked (Closure-73, Closure-126, Time-19). Two authors independently inspected, confirmed by a third."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper investigates contamination, finds 3 leaked bugs, shows GAMMA still works on perturbed versions of those bugs, and demonstrates that excluding them still yields better results than baselines (79 vs 68 for TBar, 79 vs 74 for AlphaRepair)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of an APR tool."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates software bugs from public benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "A 5-hour time limit per bug is mentioned as a fairness constraint, but no actual inference times, API costs, or per-bug costs are reported. The paper notes CodeBERT 'takes much more time' than UniXcoder without quantifying."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('one Ubuntu 18.04.3 server with two Tesla V100-SXM2 GPUs') but total GPU-hours, wall-clock time for the full experiment, or total computational cost are not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of random seed sensitivity. UniXcoder beam search is deterministic, but ChatGPT results would vary across runs and this is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It appears results are from a single run for each configuration."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Beam size is set to 250 'due to the limitation of our device' without exploring other values. No hyperparameter search was performed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses a single configuration with beam size 250 justified by device limitations. No configuration selection from multiple candidates occurs, so cherry-picking risk is minimal."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "GAMMA is compared against 9 baselines across multiple benchmarks without any statistical tests, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors reuse baseline results from prior work [18] rather than re-implementing baselines (which partially mitigates Lucic et al.'s concern), but they do not explicitly acknowledge or discuss self-comparison bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No performance-vs-compute analysis. The paper notes beam size 250 is smaller than CURE's and CoCoNuT's 1000, but does not systematically analyze the effect of compute budget on performance."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Defects4J without questioning whether bug count is a valid measure of repair capability or whether the benchmark is representative of real-world bug distributions."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. GAMMA is a deterministic pipeline (template selection → mask prediction → validation) without scaffolding or tool use."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors directly queried UniXcoder's pre-training datasets (CodeSearchNet) for overlap with Defects4J bugs, which is a stronger check than temporal analysis alone. They identified 3 overlapping bugs and verified results hold without them."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI acknowledges that perfect fault localization provides information not available in practice ('the perfect fault localization results are usually unavailable in real practice'), which is a form of feature leakage in the evaluation setup."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Defects4J bugs share structural similarities, come from the same code patterns, or are otherwise non-independent. Multiple bugs from the same project (e.g., 24 Closure bugs) could share code structure."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "The authors queried UniXcoder's pre-training datasets (2.3M NL-PL pairs, 4.1M unimodal code from CodeSearchNet) for overlap. Two authors independently performed manual inspection, confirmed by a third. They also tested GAMMA on perturbed versions of the 3 leaked bugs."
       }
     }
   },
   "claims": [
     {
       "claim": "GAMMA correctly repairs 82 bugs on Defects4J-v1.2, outperforming all compared state-of-the-art APR techniques.",
       "evidence": "Table I shows GAMMA fixes 82/101 (correct/plausible), compared to TBar (68/95), Recoder (65/112), AlphaRepair (74/109), and others. Per-project breakdowns are provided.",
       "supported": "strong"
     },
     {
       "claim": "GAMMA achieves a correct rate of 81.19% for plausible patches, higher than all baselines.",
       "evidence": "Computed from Table I: 82/101 = 81.19%. TBar is 68/95 = 71.58%, Recoder is 65/112 = 58.04%, AlphaRepair is 74/109 = 67.89%.",
       "supported": "strong"
     },
     {
       "claim": "GAMMA fixes 14 unique bugs that no other compared APR approach can fix.",
       "evidence": "Figure 2 shows the overlap Venn diagram among GAMMA, AlphaRepair, TBar, Recoder, and CURE, with 14 bugs unique to GAMMA.",
       "supported": "moderate"
     },
     {
       "claim": "GAMMA generalizes to additional benchmarks, fixing 45 bugs on Defects4J-v2.0 and 22 on QuixBugs.",
       "evidence": "Table II shows GAMMA at 45 on Defects4J-v2.0 (vs AlphaRepair 36, Recoder 11, TBar 8) and 22 on QuixBugs (vs AlphaRepair 28, CURE 26). GAMMA underperforms AlphaRepair on QuixBugs.",
       "supported": "moderate"
     },
     {
       "claim": "GAMMA is scalable to different pre-trained models: CodeBERT fixes 80 bugs, ChatGPT fixes 67 bugs on Defects4J-v1.2.",
       "evidence": "Section V-C and Figure 3 report results. The combination of all three models fixes 93 bugs total. ChatGPT underperforms UniXcoder and CodeBERT.",
       "supported": "strong"
     },
     {
       "claim": "Data leakage is minimal and does not affect the conclusion: only 3 bugs overlap with UniXcoder's pre-training data.",
       "evidence": "Section VI describes querying CodeSearchNet pre-training data, finding 3 overlapping bugs (Closure-73, Closure-126, Time-19). Excluding these, GAMMA still fixes 79 bugs vs 68 for TBar and 74 for AlphaRepair. Perturbed versions of leaked bugs are still fixable.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "All claims of outperformance are based on comparing raw bug counts without any statistical significance tests, confidence intervals, or multiple-run variance. With small differences between some tools (e.g., 82 vs 80 for CodeBERT variant), the significance of improvements is unclear."
     },
     {
       "flag": "Perfect fault localization only",
       "detail": "All experiments use perfect (ground-truth) fault localization, which is not available in practice. The authors acknowledge this but do not evaluate with realistic fault localization tools, limiting practical applicability claims."
     },
     {
       "flag": "No ablation study",
       "detail": "GAMMA has multiple novel components (template transformation to mask patterns, comment-line context addition, pre-trained model choice) but no systematic ablation isolates their individual contributions. RQ3 varies only the model, not other design decisions."
     },
     {
       "flag": "Baseline results reused from prior work",
       "detail": "Table I footnote states results are reused from AlphaRepair [18] rather than independently reproduced. The paper acknowledges 'the results of some APR tools may be different from the reported results in their published papers' due to community re-validation, introducing potential inconsistency."
     }
   ],
   "cited_papers": [
     {
       "title": "TBar: Revisiting Template-based Automated Program Repair",
       "authors": ["K. Liu", "A. Koyuncu", "D. Kim", "T. F. Bissyandé"],
       "year": 2019,
       "relevance": "State-of-the-art template-based APR tool whose fix patterns GAMMA transforms into mask patterns; primary baseline."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-shot Learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair: cloze-style APR using CodeBERT without fine-tuning, closely related zero-shot approach and key baseline."
     },
     {
       "title": "A Syntax-guided Edit Decoder for Neural Program Repair",
       "authors": ["Q. Zhu", "Z. Sun", "Y.-a. Xiao", "W. Zhang", "K. Yuan", "Y. Xiong", "L. Zhang"],
       "year": 2021,
       "relevance": "Recoder: syntax-guided learning-based APR tool, key baseline comparison."
     },
     {
       "title": "CURE: Code-aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["N. Jiang", "T. Lutellier", "L. Tan"],
       "year": 2021,
       "relevance": "NMT-based APR with code-aware search and pre-trained language model, key baseline."
     },
     {
       "title": "UniXcoder: Unified Cross-Modal Pre-Training for Code Representation",
       "authors": ["D. Guo", "S. Lu", "N. Duan", "Y. Wang", "M. Zhou", "J. Yin"],
       "year": 2022,
       "relevance": "The primary pre-trained model used by GAMMA for mask prediction, with MLM objective for code understanding and generation."
     },
     {
       "title": "CodeBERT: A Pre-Trained Model for Programming and Natural Languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang", "N. Duan", "X. Feng"],
       "year": 2020,
       "relevance": "Pre-trained code model used as alternative backbone in GAMMA's scalability experiments."
     },
     {
       "title": "CIRCLE: Continual Repair Across Programming Languages",
       "authors": ["W. Yuan", "Q. Zhang", "T. He", "C. Fang", "N. Q. V. Hung", "X. Hao", "H. Yin"],
       "year": 2022,
       "relevance": "T5-based continual learning APR framework evaluated as a baseline; co-authored by GAMMA authors."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "relevance": "Explores pre-trained models for APR with and without fine-tuning, directly related to GAMMA's investigation of LLMs for program repair."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Extensive evaluation of pre-trained models for APR including CodeX fixing 99 bugs on Defects4J-v1.2, contextualizes GAMMA's approach."
     },
     {
       "title": "CoCoNuT: Combining Context-aware Neural Translation Models Using Ensemble for Program Repair",
       "authors": ["T. Lutellier", "H. V. Pham", "L. Pang", "Y. Li", "M. Wei", "L. Tan"],
       "year": 2020,
       "relevance": "Context-aware NMT approach for APR, baseline comparison using beam size 1000."
     },
     {
       "title": "An Analysis of the Automatic Bug Fixing Performance of ChatGPT",
       "authors": ["D. Sobania", "M. Briesch", "C. Hanna", "J. Petke"],
       "year": 2023,
       "relevance": "Investigates ChatGPT's performance on QuixBugs benchmark, directly relevant to LLM-based program repair evaluation."
     },
     {
       "title": "Pre-trained Model-based Automated Software Vulnerability Repair: How Far Are We?",
       "authors": ["Q. Zhang", "C. Fang", "B. Yu", "W. Sun", "T. Zhang", "Z. Chen"],
       "year": 2023,
       "relevance": "Studies pre-trained models for security vulnerability repair, extending the GAMMA team's line of work to security domain."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "GAMMA is a released practical APR tool with source code on GitHub, usable for Java bug repair, but requires Defects4J setup and Java-specific infrastructure."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The combination of templates with pre-trained models is a natural and expected direction, not contrarian, though demonstrating zero-shot mask prediction outperforming trained NMT models is somewhat surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised; purely a software engineering tool paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; straightforward empirical comparison with existing tools."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Source code released on GitHub; a researcher could set up and run GAMMA, though it requires Java development environment and benchmark infrastructure."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses ChatGPT (recognizable) and CodeBERT (known in NLP/SE community) but from an academic lab without major brand recognition."
     }
   }
 }