ai-research-survey

scan.json (31920B)

---

 {
   "paper": {
     "title": "Revisiting Unnaturalness for Automated Program Repair in the Era of Large Language Models",
     "authors": [
       "Aidan Z.H. Yang",
       "Sophia Kolak",
       "Vincent J. Hellendoorn",
       "Ruben Martins",
       "Claire Le Goues"
     ],
     "year": 2024,
     "venue": "International Conference on Software Engineering (ICSE)",
     "arxiv_id": "2404.15236",
     "doi": "10.1109/ICSE55347.2025.00089"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM entropy is highly complementary with existing fault localization tools, improving SBFL Top-5 by up to 54% when used as a re-ranking signal. Entropy-delta, a measure of entropy change between buggy and patched code, reduces the number of patches that must be tested by a mean of 24 per bug for template-based repair (TBar). For patch correctness ranking, entropy-delta outperforms the state-of-the-art Shibboleth by 49% on Top-1 and classifies correct patches with 18% higher precision than Panther, all without requiring project-specific training data or test-suite execution.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our data, tool, and results are available and will be released as open-source' with a Zenodo URL provided (https://zenodo.org/records/10851256). A concrete archive link is given, not just a promise."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available Defects4J (V1.2 and V2.0) and curated datasets from Shibboleth and Panther replication packages. Their own replication package is at the Zenodo link."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup section is described. Model names and sizes are given (InCoder 6.7B, Starcoder 15.5B, Code-Llama2 7B) but no dependency or library versions are specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included in the paper. A Zenodo replication package is linked but the paper itself contains no 'Reproducing Results' section or commands to run."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables II, III, IV, and V are point estimates (counts or ratios). No confidence intervals, error bars, or ± notation is provided anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes multiple comparative claims (e.g., '108% improvement', '49% improvement in Top-1') based solely on comparing raw numbers without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements are reported with baseline context throughout (e.g., 'Top-1 score of 50 (108% improvement)' over SBFL's 24; 'entropy-delta improves upon Shibboleth by 49% for Top-1' from 85 to 127). Raw counts and percentages are provided together."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample sizes (395 bugs for FL, 72 for patch efficiency, 197 for patch correctness) are dictated by the Defects4J dataset and prior work, but no power analysis or justification for whether these are sufficient is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results appear to be from single runs. The LLM temperature is set to 0.5 (introducing stochasticity) but no multi-run analysis is presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: SBFL (Ochiai), TransferFL, LLMAO for fault localization; vanilla TBar for patch generation; SBFL, Shibboleth for patch ranking; PATCH-SIM, Panther for patch classification."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include TransferFL (2022), LLMAO (2023), Shibboleth (2022), and Panther (2023), which are state-of-the-art at the time of writing. SBFL/Ochiai is older but included as the standard practice baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports 24 configurations testing the contribution of each component: three LLMs, three FL tools, two filter sizes, and entropy alone. This systematically isolates the effect of entropy re-ranking across different settings."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Top-1/3/5 for fault localization, mean/median rank decrease for patch efficiency (Table III, Figure 4/5), and accuracy/precision/recall/F1 for patch classification (Table V)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated using test suite pass/fail and Top-N ranking metrics. No human evaluation of patch quality, readability, or maintainability is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The entropy approach requires no training, so there is no tuning/dev set per se. However, the choice of filter sizes (6, 10) and temperature (0.5) are not validated on a held-out set. All configurations are reported, but no explicit dev/test separation is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by Defects4J project (Chart, Closure, Lang, Math, Mockito, Time) in Tables III and IV, and Figures 4, 5, and 6."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where entropy hurts: 'we observe an overall decrease in Top-N scores using either Code-Llama2 or Starcoder entropy with a 10-filter' for SBFL, and '6-filter InCoder-entropy with TransferFL yields a Top-1 score of 57, which is a 17% decrease.' Mockito's lack of improvement is also discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: 10-filter sometimes hurts SBFL performance; entropy re-ranking shows only marginal improvements over LLMAO; entropy alone is 'only somewhat useful' for FL; entropy-delta performs worse than Panther on -recall (62.4% vs 69.6%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: '50% Top-5 score improvement over SBFL' is verified in Table II (94→145 for 6-filter Llama2, ~54%). '49% improvement in Top-1' for patch ranking matches Table IV (85→127). 'Reduce running tests for 24 patches' matches Table III mean rank decrease."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('entropy can improve fault localization') are supported by controlled ablation experiments — the same FL tools are tested with and without entropy re-ranking, isolating entropy's contribution. The choice of TBar (non-ML) specifically avoids confounding learned patterns."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Revisiting Unnaturalness for Automated Program Repair' and abstract claim about 'Automated Program Repair' are framed broadly, but all results are on Java bugs from Defects4J only. The threats section acknowledges Defects4J dependence but the framing remains broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII discusses LLM selection bias as an alternative explanation (larger models might behave differently), Defects4J data leakage (entropy scores could be contaminated), and manual labeling reliability. The paper also discusses why InCoder outperforms larger models (bidirectional training objective)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes between its proxy measures (Top-N counts, entropy-delta values) and the actual outcomes they target (fault localization accuracy, repair efficiency, patch correctness). It explicitly discusses test-suite overfitting as a gap between test-passing and true correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific open-source models with parameter counts are identified: InCoder (6.7B), Starcoder (15.5B), Code-Llama2 (7B). These are fixed-weight downloadable models, not continuously-updated APIs, so the model name and size uniquely identify the weights."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense. It computes entropy by masking code lines and using the model's log probabilities via infilling — described in Sections III-A and III-B with Figures 2 and 3. No prompt templates or instructions are sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A states 'we set the LLM temperature to 0.5' and Section III-A describes 'a sliding context window with 2048 tokens.' Model parameter counts are also reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a straightforward pipeline: compute entropy scores, then re-rank FL or patch lists. No retry logic, memory, or tool use is involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes dataset selection: which Defects4J versions are used for which RQ, which project subsets are included, and Table I shows the exact counts. The sliding context window masking approach for entropy computation is detailed in Section III-A."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats' provides substantive discussion organized into external validity, internal validity, and construct validity subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'the potential selection bias of our three selected LLMs' (external), 'Much larger LLMs (> 20 billion parameters) might have a stronger ability' (external), 'Data leakage of Defects4J as training data for our selected LLMs is possible' (external), 'manual labeling of plausible patches' (internal)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds scope: limited to 3 LLMs with infill ability, Java-only Defects4J bugs, template-based repair (TBar), and 6 of 17 Defects4J V2.0 projects for patch correctness. They state they chose not to use ML-based APR 'because our goal is a controlled evaluation of entropy without learned patterns.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J is publicly available with all source code and test suites. The Zenodo replication package contains their generated data. Prior patch datasets from Ghanbari et al. and Tian et al. are from published replication packages."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes data collection: Defects4J V1.2 for FL and patch generation (395 bugs), Defects4J V2.0 for patch correctness. Table I shows exact per-project counts. Curated patch datasets from prior work are identified by source."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from the standard Defects4J benchmark and published patch datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: bugs selected from Defects4J → entropy computed via sliding window masking → FL scores combined with entropy → patches generated by TBar → entropy-delta computed for patches → ranking/classification evaluated. Table I shows exact subset counts for each RQ."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding disclosure is present in the paper text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are clearly identified as affiliated with Carnegie Mellon University. They do not evaluate their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes what data the models were trained on (e.g., InCoder on 'public open-source repositories on GitHub and GitLab') but does not state specific training data cutoff dates for any of the three models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII explicitly states 'Data leakage of Defects4J as training data for our selected LLMs is possible' and describes two mitigation strategies: using entropy rather than direct generation, and applying entropy-delta on recently generated patches not available online. They also note GPT-4 'reports the git commit associated with the fix, implicating data leakage.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges that Defects4J code may be in training data and mitigates this by (1) using entropy rather than direct LLM code generation, (2) testing on plausible patches 'collected in 2023 (past the LLM training data cutoff of all our chosen LLMs).' The GPT-4 data leakage footnote in Section II concretely demonstrates the contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is on automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, wall-clock times, or API costs are reported. The approach requires computing entropy for every line in every file and for every generated patch, which could be expensive, but this is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, hardware specifications, or total compute budget is reported despite using three LLMs (6.7B, 7B, 15.5B parameters) for entropy computation over hundreds of bugs."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "LLM temperature is set to 0.5, introducing stochasticity in entropy computation, but no multi-seed or multi-run analysis is reported. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single values without indicating whether they are from one run or averaged over multiple."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Temperature (0.5) and context window size (2048) are reported but the paper does not describe how these were selected or what alternatives were tried. No hyperparameter search budget is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table II reports all 24 FL configurations transparently (3 LLMs × 3 FL tools × 2 filter sizes + baselines). The reader can see every result, and no configurations appear hidden. Filter sizes are justified as 'two quantities near the Top-5 that can still significantly impact Top-5 scores.'"
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparisons across 24 FL configurations, multiple projects, and multiple metrics without any correction for multiple comparisons (no Bonferroni, Holm, or similar)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors use prior tools' replication packages directly rather than re-implementing baselines, which partially mitigates this concern. However, they do not explicitly discuss or acknowledge author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "InCoder (6.7B) outperforms larger Starcoder (15.5B) and Code-Llama2 (7B), which is noted, but no systematic compute-vs-performance analysis is presented. The computational cost of entropy computation itself is not reported."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Defects4J is a valid proxy for real-world APR. It is used because prior tools are compatible with it, but its representativeness of real-world bugs is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The approach computes entropy scores and re-ranks existing FL/patch lists without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states that the 2,147 plausible patches used for RQ3 were 'collected in 2023 (past the LLM training data cutoff of all our chosen LLMs),' directly addressing temporal leakage for patch correctness. For FL and RQ2, the concern is acknowledged in threats to validity."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper argues that using entropy rather than direct generation reduces leakage risk, but does not analyze whether the entropy signal itself could be inflated by the model having seen Defects4J code during training. The models' familiarity with the codebase could systematically affect entropy values."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data and Defects4J test examples share structural similarities (e.g., same repositories, similar code patterns). The open-source training data for all three models likely includes Java repositories from which Defects4J bugs were drawn."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Two concrete methods are used: (1) temporal separation — patch correctness dataset collected in 2023 after all models' training cutoffs, and (2) anecdotal detection — demonstrating GPT-4 data leakage by showing it 'reports the git commit associated with the fix.' The approach also uses entropy rather than direct generation as a structural mitigation."
       }
     }
   },
   "claims": [
     {
       "claim": "Entropy re-ranking improves SBFL Top-5 fault localization by up to 54% (from 94 to 145 bugs)",
       "evidence": "Table II shows SBFL alone achieves Top-5=94; 6-filter with Llama2 entropy achieves Top-5=145. InCoder entropy with 6-filter achieves Top-1=55 (129% over SBFL's 24). Section V RQ1.",
       "supported": "strong"
     },
     {
       "claim": "Entropy-delta reduces the mean number of patches tested by 24 per bug for TBar",
       "evidence": "Table III shows entropy-delta improves 60 of 72 patch rankings with mean rank decrease of 24. Figure 5 shows median patches tested decreases across most projects. Section V RQ2.",
       "supported": "strong"
     },
     {
       "claim": "Entropy-delta outperforms Shibboleth by 49% on Top-1 for patch correctness ranking",
       "evidence": "Table IV: entropy-delta Top-1=127 vs Shibboleth Top-1=85 across 1,290 patches on Defects4J V2.0. Top-2 also improves: 165 vs 130 (27% improvement). Section V RQ3.",
       "supported": "strong"
     },
     {
       "claim": "Entropy-delta classifies correct patches with 18% higher precision than Panther",
       "evidence": "Table V: entropy-delta precision=0.900 vs Panther precision=0.760 on 2,147 plausible patches. F1 also improves: 0.824 vs 0.750 (10% improvement). Section V RQ3.",
       "supported": "strong"
     },
     {
       "claim": "InCoder (6.7B) is the most effective LLM for entropy-based fault localization despite being the smallest model",
       "evidence": "Table II: InCoder consistently achieves the highest Top-N scores across configurations (e.g., entropy-alone Top-5=116 vs Starcoder 55 vs Llama2 41). Attributed to InCoder's bidirectional infilling training objective. Section V RQ1.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs can complement APR while minimizing test-suite overfitting and data leakage",
       "evidence": "The entropy approach avoids direct code generation (reducing leakage risk) and does not require test execution for ranking (reducing overfitting). However, leakage mitigation is argued structurally rather than empirically verified. Sections I, VII, VIII.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (e.g., '108% improvement', '49% improvement') are based on comparing raw counts without any significance tests. With the stochasticity introduced by temperature=0.5, results could vary across runs, but no uncertainty quantification is provided."
     },
     {
       "flag": "Single benchmark only",
       "detail": "All experiments use only Defects4J (Java). The title and abstract frame contributions for 'Automated Program Repair' broadly, but generalization to other languages, bug types, or benchmarks is untested."
     },
     {
       "flag": "Contamination risk not fully mitigated",
       "detail": "All three LLMs were trained on open-source code that likely includes Defects4J repositories. While entropy rather than direct generation is used, the entropy values themselves could be affected by the model's familiarity with these specific codebases. Training cutoff dates are not stated."
     },
     {
       "flag": "No compute cost reporting",
       "detail": "Computing entropy for every line in every file across 395 bugs using three LLMs (6.7B-15.5B parameters) is computationally expensive, but no wall-clock time, GPU hours, or costs are reported. This matters for practical applicability."
     }
   ],
   "cited_papers": [
     {
       "title": "Tbar: Revisiting template-based automated program repair",
       "authors": ["K. Liu", "A. Koyuncu", "D. Kim", "T. F. Bissyandé"],
       "year": 2019,
       "relevance": "Primary template-based APR tool used as the patch generation baseline in efficiency experiments."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["C. S. Xia", "Y. Wei", "L. Zhang"],
       "year": 2023,
       "relevance": "Directly applied LLMs for APR and found they can suggest multi-line fixes more accurately than prior APR tools."
     },
     {
       "title": "Large language models for test-free fault localization",
       "authors": ["A. Z. Yang", "R. Martins", "C. Le Goues", "V. J. Hellendoorn"],
       "year": 2023,
       "arxiv_id": "2310.01726",
       "relevance": "LLMAO: state-of-the-art LLM-based fault localization tool used as a baseline in this paper."
     },
     {
       "title": "Patch correctness assessment in automated program repair based on the impact of patches on production and test code",
       "authors": ["A. Ghanbari", "A. Marcus"],
       "year": 2022,
       "relevance": "Shibboleth: state-of-the-art patch ranking tool and source of the 1,290 patch dataset used for evaluation."
     },
     {
       "title": "The best of both worlds: Combining learned embeddings with engineered features for accurate prediction of correct patches",
       "authors": ["H. Tian", "K. Liu", "Y. Li"],
       "year": 2023,
       "relevance": "Panther: state-of-the-art patch classifier using BERT embeddings and source of the 2,147 patch dataset."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper establishing LLM code generation evaluation methodology."
     },
     {
       "title": "InCoder: A generative model for code infilling and synthesis",
       "authors": ["D. Fried", "A. Aghajanyan", "J. Lin"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Primary LLM used for entropy computation; its bidirectional infilling training proved most effective for repair tasks."
     },
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "One of three LLMs evaluated for entropy-based program repair."
     },
     {
       "title": "On the naturalness of buggy code",
       "authors": ["B. Ray", "V. Hellendoorn", "S. Godhane"],
       "year": 2016,
       "relevance": "Foundational work establishing that n-gram language models find buggy code more surprising — the hypothesis this paper revisits with modern LLMs."
     },
     {
       "title": "Leak, cheat, repeat: Data contamination and evaluation malpractices in closed-source LLMs",
       "authors": ["S. Balloccu", "P. Schmidtová", "M. Lango", "O. Dušek"],
       "year": 2024,
       "arxiv_id": "2402.03927",
       "relevance": "Directly relevant to data contamination concerns in LLM evaluation."
     },
     {
       "title": "A large-scale empirical review of patch correctness checking approaches",
       "authors": ["J. Yang", "Y. Wang", "Y. Lou", "M. Wen", "L. Zhang"],
       "year": 2023,
       "relevance": "Found that naturalness-based techniques outperform static techniques for patch correctness, motivating this paper's entropy approach."
     },
     {
       "title": "Improving fault localization and program repair with deep semantic features and transferred knowledge",
       "authors": ["X. Meng", "X. Wang", "H. Zhang"],
       "year": 2022,
       "relevance": "TransferFL: state-of-the-art transfer-learning-based fault localization tool used as baseline."
     },
     {
       "title": "On the efficiency of test suite based program repair: A systematic assessment of 16 automated repair systems for java programs",
       "authors": ["K. Liu", "S. Wang", "A. Koyuncu"],
       "year": 2020,
       "relevance": "Established patch evaluation count as a hardware-independent efficiency metric for APR, used in this paper's RQ2."
     },
     {
       "title": "Is the cure worse than the disease? overfitting in automated program repair",
       "authors": ["E. K. Smith", "E. T. Barr", "C. Le Goues", "Y. Brun"],
       "year": 2015,
       "doi": "10.1145/2786805.2786825",
       "relevance": "Foundational work on test-suite overfitting in APR — the key problem that entropy-delta aims to mitigate."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Entropy-delta could be integrated into existing APR tools to reduce test overhead and improve patch ranking, with a modified TBar released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that the smallest model (InCoder 6.7B) outperforms larger ones for entropy-based FL is mildly surprising but consistent with prior work on infilling models."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the work focuses on improving automated program repair."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; the paper proposes a complementary technique rather than challenging existing approaches."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A Zenodo replication package is provided but requires Java/Defects4J setup and local LLM inference — not trivially runnable."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are at Carnegie Mellon (well-known in SE research) and published at ICSE, but no product branding or famous-lab association."
     }
   }
 }