ai-research-survey

scan.json (31605B)

---

 {
   "paper": {
     "title": "Extracting Fix Ingredients using Language Models",
     "authors": [
       "Julian Aron Prenner",
       "Romain Robbes"
     ],
     "year": 2025,
     "venue": "2025 IEEE/ACM Second International Conference on AI Foundation Models and Software Engineering (Forge)",
     "arxiv_id": "2503.04214",
     "doi": "10.1109/Forge66646.2025.00028"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "observational"],
   "key_findings": "Identifier ingredients are prevalent in program repair (44% of TSSB-3M, 85% of Defects4J fixes) and 39-51% fall outside a typical model input window. ScanFix, combining a scanner model with a repair model, yields 7-31% relative improvement over no-ingredient baselines for out-of-window bugs. However, simply expanding the input window to 5120 tokens outperforms ScanFix, illustrating Sutton's bitter lesson. Oracle ingredients show far higher performance (51.99%), indicating that better extraction techniques could still be valuable.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and scripts used in this work are provided online[12]' with reference [12] pointing to a GitHub repository (https://github.com/giganticode/llm_ingredient_extraction)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: TSSB-3M and Defects4J 2.0. Both are standard benchmarks accessible to other researchers. The replication package includes scripts for data processing."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model architectures (CodeT5-small 60M, StarEncoder 125M) and mentions tools like TreeSitter and Pygments, but does not provide a requirements.txt, Dockerfile, or detailed library version specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While a replication package URL is provided, the paper itself does not include step-by-step reproduction instructions, a README with commands, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 95% confidence interval bands on Figures 5, 6, 7, and 10. For example, 'Error bands indicate a 95% confidence interval' is stated for multiple figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No formal statistical significance tests (p-values, t-tests, etc.) are reported. The paper uses 'significant' informally (e.g., 'a significant increase in repair success') without formal testing. Comparative claims rely on point estimates and confidence interval overlap."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both absolute and relative improvements with baseline context. For example, 'absolute performance increase of 2.55% and a relative improvement of roughly 7%' and 'a relative improvement of 31.5% (abs. 5.9%).' Table II provides full numeric context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A (Datasets) explicitly justifies dataset size choices: 'Defects4J has only around 800 bugs... each individual bug would have a very large weight. In contrast, our evaluation set for TSSB is between 80 times (RQ3) and 250 times larger (RQ4), which ensures much more robust results.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table II are single-run point estimates with no standard deviation, variance, or multiple-run analysis. The 95% CI bands in figures show variation across bugs, not across experimental runs with different seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes six baselines: perfect ingredients, perfect file-level ingredients, perfect recall/low precision, naive ingredients, no ingredients, and large context. For Defects4J RQ2, results from 15 published APR/NPR tools are compared."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "For Defects4J analysis, baselines include recent tools like FitRepair, TARE, RAP-Gen (2022-2023). The large-context baseline provides a fair apples-to-apples comparison using the same model architecture."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically varies: scanner variants (All vs OOW), extraction thresholds (0.05, 0.5, 0.95), ingredient sources (none, naive, scanned, perfect, large context). Table II presents results for all combinations, showing each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The scanner model is evaluated with precision, recall, and F1 (Section V-B). Repair is evaluated with exact match. The ingredient analysis uses cover, distance, and frequency metrics. Multiple perspectives on the same problem."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of repair outputs is performed. Evaluation is entirely automated using exact match against ground-truth fixes."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table I explicitly documents disjoint training and evaluation splits: 510,851 training / 85,776 evaluation for RQ2; 274,776 / 85,776 for scanner (RQ3); 236,075 / 257,316 for repair (RQ4). The paper also ensures disjoint scanner and repair training sets to avoid data leakage."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by fix ingredient count (Figures 5, 6), ingredient distance (Figures 7, 10), in-window vs out-of-window (Figure 6, Table II columns), ingredient frequency (Figure 7), and context level (Figure 3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where the approach fails: ScanFix is outperformed by large context (Section VI-B), performance degrades with multiple ingredients ('the complexity of arranging the ingredients dominates'), and the scanner has modest extraction performance."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper openly reports that ScanFix is outperformed by the large-context baseline (Section VI-B) and frames this as 'A case of Sutton's bitter lesson' (Section VII-B). The scanner's modest precision/recall is honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'relative improvements of up to 31%' matches Table II (31.5% for OOW winout); 'outperformed by a model with a large input window' matches Table II (39.28% vs 37.27%); 'augmenting with ground-truth fix leads to even better results' matches Table II (51.99%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('augmenting repair model input with ingredients yields improvements') are supported by controlled experimental comparisons: same model architecture, same training data, varying only the ingredient augmentation. The ablation-style design (adding/removing ingredients) is adequate for these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'a dataset of 85,000 Python bugs.' Section VII-A explicitly bounds scope: single model architecture, file-level scanning only for TSSB-3M, identifier ingredients only (not literals or compound ingredients), lexical analysis only."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-B discusses Sutton's bitter lesson as an alternative framing. The paper considers that large context windows might solve the problem entirely, that memorization in models could explain some performance, and that ingredient frequency in training data affects success (Section IV-D)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges using exact match as a proxy: 'bugs in TSSB-3M are not executable (and lack tests) we resort to exact match to assess whether a generated patch is correct or not' (Section IV-B). This clearly distinguishes the measurement proxy from actual correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'CodeT5 small variant with 60M parameters' (ref [37]) for the repair model and 'BigCode's pre-trained StarEncoder model with roughly 125M parameters' (refs [38, 39]) for the scanner model. These are specific, identifiable model checkpoints."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper's main experiments use fine-tuned seq2seq and token-classification models, not prompting. The ChatGPT example in Section I is a motivational illustration, not a systematic experiment."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Hyperparameters are reported in detail: repair model uses learning rate 1×10⁻⁴, 4 epochs, batch size 12 accumulated over 2 steps, beam search with 5 beams (Section IV-B). Scanner uses learning rate 6×10⁻⁵, 4 epochs, batch size 30 accumulated over 3 steps (Section V-A)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a standard two-stage pipeline of fine-tuned models (scanner followed by repair model)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents preprocessing in detail: TreeSitter-based multi-line string/comment splitting, commit-hash deduplication (reducing TSSB-3M from 3M+ to ~900K), filtering of encoding issues (3 Defects4J bugs omitted), and context window construction (18 lines before, 12 after)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A 'Limitations' is a substantial dedicated section covering seven specific limitations: potential software bugs, non-identifier ingredients, lexical analysis limitations, single model architecture, LLM impact, dataset constraints, and file-level scanning limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'We use a single model architecture' with explanation that 'repeating experiments using a second or even third model architecture would have exceeded our computational budget'; 'we were not able to use Defects4J... due to its small size'; lexical analysis 'ignores an identifier's syntactic function.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A explicitly states what was NOT tested: non-identifier ingredients ('Literals, compound ingredients... are not studied'), project-level scanning for TSSB-3M ('mining for project-level identifiers impractical'), LLM evaluation ('we decided not to focus on LLMs'), and scope/accessibility analysis."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Both underlying datasets (TSSB-3M and Defects4J 2.0) are publicly available. The replication package [12] provides the code and scripts used for data processing, enabling independent verification from the raw data sources."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes data collection: TSSB-3M file-level context mined via GitHub API using commit hashes ('this step takes several weeks'), Defects4J uses 'relevant classes' bug properties. Deduplication and filtering criteria are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (TSSB-3M, Defects4J)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: mining (GitHub API) → preprocessing (TreeSitter) → deduplication (commit hashes, reducing 3M+ to ~900K) → filtering (encoding issues) → ingredient extraction (Pygments) → dataset splitting (Table I shows exact split sizes). The deduplication step's quantitative impact is stated."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This study has received financial support from the French State in the framework of the Investments for the Future programme IdEx université de Bordeaux.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Julian Aron Prenner at Free University of Bozen-Bolzano, Romain Robbes at Univ. Bordeaux/CNRS/Bordeaux INP/LaBRI. No commercial product is being evaluated, so no product-affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The French State's IdEx program is a general academic funding source with no financial stake in whether ScanFix outperforms baselines."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the pre-trained CodeT5 or StarEncoder models. Without cutoff dates, it is impossible to assess whether TSSB-3M or Defects4J code appeared in pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions memorization concerns in Section VII-A ('Previous work expressed concerns of memorization issues in APR') and uses a small model to minimize this, but does not analyze whether specific test examples overlap with pre-training data for CodeT5 or StarEncoder."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TSSB-3M contains bugs from public GitHub repositories that could be in CodeT5's and StarEncoder's pre-training data (both trained on open-source code). The paper does not address this contamination risk beyond noting memorization concerns and using a smaller model."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments use automated bug datasets and model evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or API cost is reported. The paper mentions VRAM constraints ('fully exhausting our VRAM budget') but does not quantify inference time or per-example cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions exceeding computational budget as a constraint ('would have exceeded our computational budget') but does not state the actual budget: no GPU hours, hardware specifications, or training time are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single training runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of training runs is not stated. The paper reports beam search with 5 beams for candidate generation but does not state how many training runs produced the reported results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Specific hyperparameters are reported but no search budget is disclosed. There is no indication of how the learning rates, batch sizes, or number of epochs were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper transparently reports results for all scanner variants (All, OOW) at all thresholds (0.05, 0.5, 0.95) and all baselines in Table II. No cherry-picking — all configurations are shown."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The baselines (no ingredients, naive ingredients, large context) are all implemented by the authors. For Defects4J RQ2, published results from other tools are used. The paper does not acknowledge the bias of evaluating their own system against their own baseline implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The large-context model uses 5× more input tokens (5120 vs 1024) and the paper discusses quadratic attention cost, but performance is not formally reported as a function of compute budget. No FLOPs or training time comparisons are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges exact match as a proxy ('we resort to exact match to assess whether a generated patch is correct or not') but does not discuss whether TSSB-3M's single-statement Python bugs actually measure general program repair capability or whether exact match is a valid construct for correctness."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The models are fine-tuned seq2seq and token-classification models, not scaffold-dependent agents."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeT5 or StarEncoder pre-training data temporally overlaps with TSSB-3M or Defects4J bug data. The temporal relationship between pre-training data collection and benchmark creation is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For instance, the ground-truth fix is used to determine positive labels for the scanner training, but potential feature leakage through the input structure is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Commit-hash deduplication removes exact duplicates, but the paper does not discuss whether train and test examples may share repositories, authors, or structurally similar code patterns that could inflate performance."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "Commit-hash deduplication and disjoint train/test splits (Table I) are good practices but are not leakage detection methods in the formal sense (no canary strings, membership inference, n-gram overlap analysis, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "Identifier ingredients are prevalent: 44% of TSSB-3M fixes and 85% of Defects4J fixes require at least one identifier ingredient.",
       "evidence": "Section III-C, analysis of ~900K TSSB-3M bugs and 832 Defects4J bugs. Figure 3 shows cover distributions across context levels.",
       "supported": "strong"
     },
     {
       "claim": "39-51% of identifier ingredients fall outside a typical 30-line input window context.",
       "evidence": "Section III-C, Figure 3. Window cover is 61% for Defects4J and 49% for TSSB-3M, meaning 39% and 51% respectively are uncovered.",
       "supported": "strong"
     },
     {
       "claim": "Fix ingredient count and distance from bug location significantly affect repair success across multiple APR/NPR tools.",
       "evidence": "Section IV-C/D. Figure 5 shows downward trend across 15 tools on Defects4J. Figures 6 and 7 show similar patterns on TSSB-3M. In-window ingredients yield higher success than out-of-window.",
       "supported": "strong"
     },
     {
       "claim": "The scanner model extracts fix ingredients with 17-48% recall and 7-45% precision depending on threshold.",
       "evidence": "Section V-B, Figure 9. All variant at t=0.5: recall 47.66%, precision 7.05%. At t=0.95: recall 17.01%, precision 45.09%.",
       "supported": "moderate"
     },
     {
       "claim": "ScanFix yields 7-31% relative improvement over no-ingredient baseline, especially for out-of-window ingredients.",
       "evidence": "Section VI-B, Table II. All bugs: 37.27% vs 34.72% (7.3% relative). Bugs with winout ingredients: 24.56% vs 18.68% (31.5% relative, OOW variant).",
       "supported": "strong"
     },
     {
       "claim": "A large-context baseline (5120 tokens) outperforms all ScanFix variants without any ingredient augmentation.",
       "evidence": "Table II: large context achieves 39.28% on all bugs vs best ScanFix 37.27%. Section VII-B frames this as Sutton's bitter lesson.",
       "supported": "strong"
     },
     {
       "claim": "Perfect (oracle) ingredients lead to 51.99% exact match, far exceeding all other approaches, showing potential for improved extraction.",
       "evidence": "Table II: perfect ingredients 51.99% vs large context 39.28% vs best ScanFix 37.27%. Also, 'perfect recall, low precision' baseline at 40.29% shows repair model tolerates noisy ingredients.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model architecture",
       "detail": "All experiments use a single model family (CodeT5-small for repair, StarEncoder for scanner). The authors acknowledge this limitation and cite computational budget constraints, but results may not generalize to other architectures."
     },
     {
       "flag": "Exact match evaluation only",
       "detail": "TSSB-3M evaluation relies solely on exact match with ground-truth fixes. Semantically correct but textually different patches are counted as failures, potentially underestimating true repair rates for all approaches equally."
     },
     {
       "flag": "No formal significance testing",
       "detail": "Comparative claims between ScanFix variants and baselines are made based on point estimates in Table II without statistical significance tests. With single-run results, it is unclear whether observed differences are statistically meaningful."
     },
     {
       "flag": "No multi-seed evaluation",
       "detail": "Results appear to come from single training runs without seed variation. Henderson et al. (2018) showed that results can vary substantially across seeds, which is acknowledged in the field but not addressed here."
     }
   ],
   "cited_papers": [
     {
       "title": "CoCoNuT: Combining context-aware neural translation models using ensemble for program repair",
       "authors": ["T. Lutellier", "H. V. Pham", "L. Pang", "Y. Li", "M. Wei", "L. Tan"],
       "year": 2020,
       "relevance": "Neural program repair approach using ensemble models, one of the NPR tools benchmarked in this study."
     },
     {
       "title": "Can OpenAI's Codex Fix Bugs? An Evaluation on QuixBugs",
       "authors": ["J. A. Prenner", "H. Babii", "R. Robbes"],
       "year": 2022,
       "relevance": "Early evaluation of LLMs (Codex) for automated program repair, raises memorization concerns relevant to contamination."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "relevance": "Comprehensive study of how code language models affect automated program repair performance."
     },
     {
       "title": "The plastic surgery hypothesis",
       "authors": ["E. T. Barr", "Y. Brun", "P. Devanbu", "M. Harman", "F. Sarro"],
       "year": 2014,
       "relevance": "Foundational hypothesis that correct fixes can be crafted from existing code elements, central premise of this work."
     },
     {
       "title": "Out of context: How important is local context in neural program repair?",
       "authors": ["J. A. Prenner", "R. Robbes"],
       "year": 2023,
       "arxiv_id": "2312.04986",
       "relevance": "Directly relevant predecessor studying how context window size affects neural program repair, informs this paper's context design."
     },
     {
       "title": "SequenceR: Sequence-to-Sequence Learning for End-to-End Program Repair",
       "authors": ["Z. Chen", "S. Kommrusch", "M. Tufano", "L.-N. Pouchet", "D. Poshyvanyk", "M. Monperrus"],
       "year": 2021,
       "relevance": "Early neural program repair using LSTM with input augmentation (field variables, method signatures), directly related to ingredient augmentation approach."
     },
     {
       "title": "RAP-Gen: Retrieval-Augmented Patch Generation with CodeT5 for Automatic Program Repair",
       "authors": ["W. Wang", "Y. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2023,
       "relevance": "Retrieval-augmented approach to program repair, alternative to ingredient scanning for incorporating project-level context."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Y. Zhang", "H. Ruan", "Z. Fan", "A. Roychoudhury"],
       "year": 2024,
       "relevance": "Autonomous agent for program repair using retrieval-augmented generation, relevant to agentic AI repair approaches."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["I. Bouzenia", "P. Devanbu", "M. Pradel"],
       "year": 2024,
       "relevance": "LLM-based autonomous agent for program repair, represents the agentic approach to automated repair."
     },
     {
       "title": "Revisiting the Plastic Surgery Hypothesis via Large Language Models",
       "authors": ["C. S. Xia", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "relevance": "Uses Levenshtein-based identifier extraction and project-specific fine-tuning (FitRepair) for LLM-based repair, directly comparable approach."
     },
     {
       "title": "Less training, more repairing please: Revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair: zero-shot LLM-based program repair, one of the NPR tools analyzed in the Defects4J evaluation."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. H. Hoi"],
       "year": 2021,
       "relevance": "Foundation model used for the repair model in this work; identifier-aware pre-training is directly relevant to the ingredient extraction task."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Rozière", "J. Gehring", "F. Gloeckle"],
       "year": 2024,
       "relevance": "Large-context code LLM (16K+ tokens) representing the trend of expanding input windows that may make ingredient extraction obsolete."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Research contribution to APR methodology; not directly usable as a practitioner tool, though the replication code is available."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 'bitter lesson' framing—showing that simply increasing context window outperforms a more sophisticated approach—is mildly contrarian within the APR community."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code released on GitHub but requires training infrastructure and dataset preparation; not easily demoed."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Free University of Bozen-Bolzano and University of Bordeaux; not well-known AI labs."
     }
   }
 }