ai-research-survey

scan-v4.json (34600B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Extracting Fix Ingredients using Language Models",
     "authors": [
       "Julian Aron Prenner",
       "Romain Robbes"
     ],
     "year": 2025,
     "venue": "2025 IEEE/ACM Second International Conference on AI Foundation Models and Software Engineering (Forge)",
     "arxiv_id": "2503.04214",
     "doi": "10.1109/Forge66646.2025.00028"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'relative improvements of up to 31%' matches Table II (31.5% for OOW winout); 'outperformed by a model with a large input window' matches Table II (39.28% vs 37.27%); 'augmenting with ground-truth fix leads to even better results' matches Table II (51.99%).",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('augmenting repair model input with ingredients yields improvements') are supported by controlled experimental comparisons: same model architecture, same training data, varying only the ingredient augmentation. The ablation-style design (adding/removing ingredients) is adequate for these claims.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'a dataset of 85,000 Python bugs.' Section VII-A explicitly bounds scope: single model architecture, file-level scanning only for TSSB-3M, identifier ingredients only (not literals or compound ingredients), lexical analysis only.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-B discusses Sutton's bitter lesson as an alternative framing. The paper considers that large context windows might solve the problem entirely, that memorization in models could explain some performance, and that ingredient frequency in training data affects success (Section IV-D).",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges using exact match as a proxy: 'bugs in TSSB-3M are not executable (and lack tests) we resort to exact match to assess whether a generated patch is correct or not' (Section IV-B). This clearly distinguishes the measurement proxy from actual correctness.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A 'Limitations' is a substantial dedicated section covering seven specific limitations: potential software bugs, non-identifier ingredients, lexical analysis limitations, single model architecture, LLM impact, dataset constraints, and file-level scanning limitations.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'We use a single model architecture' with explanation that 'repeating experiments using a second or even third model architecture would have exceeded our computational budget'; 'we were not able to use Defects4J... due to its small size'; lexical analysis 'ignores an identifier's syntactic function.'",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A explicitly states what was NOT tested: non-identifier ingredients ('Literals, compound ingredients... are not studied'), project-level scanning for TSSB-3M ('mining for project-level identifiers impractical'), LLM evaluation ('we decided not to focus on LLMs'), and scope/accessibility analysis.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section states: 'This study has received financial support from the French State in the framework of the Investments for the Future programme IdEx université de Bordeaux.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Julian Aron Prenner at Free University of Bozen-Bolzano, Romain Robbes at Univ. Bordeaux/CNRS/Bordeaux INP/LaBRI. No commercial product is being evaluated, so no product-affiliation conflict exists.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The French State's IdEx program is a general academic funding source with no financial stake in whether ScanFix outperforms baselines.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Fix ingredients, identifier ingredients, ingredient cover, and the full taxonomy (fixall, winin, winout, mthin, mthout, filein, fileout, projin, projout) are formally defined in Section III.B.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly states two contributions: an empirical study of identifier ingredient prevalence and impact (RQ1-2), and the ScanFix system combining a scanner and repair model (RQ3-4).",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section II provides detailed comparison to SequenceR, RewardRepair, FitRepair, RAP-Gen, SelfAPR, and others, explicitly explaining how ScanFix differs (neural scanner vs. static/lexical extraction, fine-tuned repair model).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper states 'Code and scripts used in this work are provided online[12]' with reference [12] pointing to a GitHub repository (https://github.com/giganticode/llm_ingredient_extraction).",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The paper uses two publicly available datasets: TSSB-3M and Defects4J 2.0. Both are standard benchmarks accessible to other researchers. The replication package includes scripts for data processing.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper specifies model architectures (CodeT5-small 60M, StarEncoder 125M) and mentions tools like TreeSitter and Pygments, but does not provide a requirements.txt, Dockerfile, or detailed library version specifications sufficient to recreate the environment.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "While a replication package URL is provided, the paper itself does not include step-by-step reproduction instructions, a README with commands, or a 'Reproducing Results' section.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports 95% confidence interval bands on Figures 5, 6, 7, and 10. For example, 'Error bands indicate a 95% confidence interval' is stated for multiple figures.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No formal statistical significance tests (p-values, t-tests, etc.) are reported. The paper uses 'significant' informally (e.g., 'a significant increase in repair success') without formal testing. Comparative claims rely on point estimates and confidence interval overlap.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper reports both absolute and relative improvements with baseline context. For example, 'absolute performance increase of 2.55% and a relative improvement of roughly 7%' and 'a relative improvement of 31.5% (abs. 5.9%).' Table II provides full numeric context.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": true,
           "justification": "Section VII-A (Datasets) explicitly justifies dataset size choices: 'Defects4J has only around 800 bugs... each individual bug would have a very large weight. In contrast, our evaluation set for TSSB is between 80 times (RQ3) and 250 times larger (RQ4), which ensures much more robust results.'",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "The main results in Table II are single-run point estimates with no standard deviation, variance, or multiple-run analysis. The 95% CI bands in figures show variation across bugs, not across experimental runs with different seeds.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "The paper includes six baselines: perfect ingredients, perfect file-level ingredients, perfect recall/low precision, naive ingredients, no ingredients, and large context. For Defects4J RQ2, results from 15 published APR/NPR tools are compared.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "For Defects4J analysis, baselines include recent tools like FitRepair, TARE, RAP-Gen (2022-2023). The large-context baseline provides a fair apples-to-apples comparison using the same model architecture.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "The paper systematically varies: scanner variants (All vs OOW), extraction thresholds (0.05, 0.5, 0.95), ingredient sources (none, naive, scanned, perfect, large context). Table II presents results for all combinations, showing each component's contribution.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The scanner model is evaluated with precision, recall, and F1 (Section V-B). Repair is evaluated with exact match. The ingredient analysis uses cover, distance, and frequency metrics. Multiple perspectives on the same problem.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "No human evaluation of repair outputs is performed. Evaluation is entirely automated using exact match against ground-truth fixes.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Table I explicitly documents disjoint training and evaluation splits: 510,851 training / 85,776 evaluation for RQ2; 274,776 / 85,776 for scanner (RQ3); 236,075 / 257,316 for repair (RQ4). The paper also ensures disjoint scanner and repair training sets to avoid data leakage.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by fix ingredient count (Figures 5, 6), ingredient distance (Figures 7, 10), in-window vs out-of-window (Figure 6, Table II columns), ingredient frequency (Figure 7), and context level (Figure 3).",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper discusses where the approach fails: ScanFix is outperformed by large context (Section VI-B), performance degrades with multiple ingredients ('the complexity of arranging the ingredients dominates'), and the scanner has modest extraction performance.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "The paper openly reports that ScanFix is outperformed by the large-context baseline (Section VI-B) and frames this as 'A case of Sutton's bitter lesson' (Section VII-B). The scanner's modest precision/recall is honestly reported.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "The paper specifies 'CodeT5 small variant with 60M parameters' (ref [37]) for the repair model and 'BigCode's pre-trained StarEncoder model with roughly 125M parameters' (refs [38, 39]) for the scanner model. These are specific, identifiable model checkpoints.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": false,
           "answer": false,
           "justification": "The paper's main experiments use fine-tuned seq2seq and token-classification models, not prompting. The ChatGPT example in Section I is a motivational illustration, not a systematic experiment.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Hyperparameters are reported in detail: repair model uses learning rate 1×10⁻⁴, 4 epochs, batch size 12 accumulated over 2 steps, beam search with 5 beams (Section IV-B). Scanner uses learning rate 6×10⁻⁵, 4 epochs, batch size 30 accumulated over 3 steps (Section V-A).",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used. The approach is a standard two-stage pipeline of fine-tuned models (scanner followed by repair model).",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section III-A documents preprocessing in detail: TreeSitter-based multi-line string/comment splitting, commit-hash deduplication (reducing TSSB-3M from 3M+ to ~900K), filtering of encoding issues (3 Defects4J bugs omitted), and context window construction (18 lines before, 12 after).",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "Both underlying datasets (TSSB-3M and Defects4J 2.0) are publicly available. The replication package [12] provides the code and scripts used for data processing, enabling independent verification from the raw data sources.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section III-A describes data collection: TSSB-3M file-level context mined via GitHub API using commit hashes ('this step takes several weeks'), Defects4J uses 'relevant classes' bug properties. Deduplication and filtering criteria are specified.",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. Data sources are standard benchmarks (TSSB-3M, Defects4J).",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The pipeline is documented: mining (GitHub API) → preprocessing (TreeSitter) → deduplication (commit hashes, reducing 3M+ to ~900K) → filtering (encoding issues) → ingredient extraction (Pygments) → dataset splitting (Table I shows exact split sizes). The deduplication step's quantitative impact is stated.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not state the training data cutoff dates for the pre-trained CodeT5 or StarEncoder models. Without cutoff dates, it is impossible to assess whether TSSB-3M or Defects4J code appeared in pre-training data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions memorization concerns in Section VII-A ('Previous work expressed concerns of memorization issues in APR') and uses a small model to minimize this, but does not analyze whether specific test examples overlap with pre-training data for CodeT5 or StarEncoder.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "TSSB-3M contains bugs from public GitHub repositories that could be in CodeT5's and StarEncoder's pre-training data (both trained on open-source code). The paper does not address this contamination risk beyond noting memorization concerns and using a smaller model.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study. All experiments use automated bug datasets and model evaluations.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "No inference cost, latency, or API cost is reported. The paper mentions VRAM constraints ('fully exhausting our VRAM budget') but does not quantify inference time or per-example cost.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions exceeding computational budget as a constraint ('would have exceeded our computational budget') but does not state the actual budget: no GPU hours, hardware specifications, or training time are reported.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results appear to be from single training runs.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": false,
           "justification": "The number of training runs is not stated. The paper reports beam search with 5 beams for candidate generation but does not state how many training runs produced the reported results.",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "Specific hyperparameters are reported but no search budget is disclosed. There is no indication of how the learning rates, batch sizes, or number of epochs were selected.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": true,
           "justification": "The paper transparently reports results for all scanner variants (All, OOW) at all thresholds (0.05, 0.5, 0.95) and all baselines in Table II. No cherry-picking — all configurations are shown.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": false,
           "answer": false,
           "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The baselines (no ingredients, naive ingredients, large context) are all implemented by the authors. For Defects4J RQ2, published results from other tools are used. The paper does not acknowledge the bias of evaluating their own system against their own baseline implementations.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": false,
           "justification": "The large-context model uses 5× more input tokens (5120 vs 1024) and the paper discusses quadratic attention cost, but performance is not formally reported as a function of compute budget. No FLOPs or training time comparisons are provided.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper acknowledges exact match as a proxy ('we resort to exact match to assess whether a generated patch is correct or not') but does not discuss whether TSSB-3M's single-statement Python bugs actually measure general program repair capability or whether exact match is a valid construct for correctness.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is involved. The models are fine-tuned seq2seq and token-classification models, not scaffold-dependent agents.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether CodeT5 or StarEncoder pre-training data temporally overlaps with TSSB-3M or Defects4J bug data. The temporal relationship between pre-training data collection and benchmark creation is not addressed.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the evaluation setup leaks information. For instance, the ground-truth fix is used to determine positive labels for the scanner training, but potential feature leakage through the input structure is not discussed.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Commit-hash deduplication removes exact duplicates, but the paper does not discuss whether train and test examples may share repositories, authors, or structurally similar code patterns that could inflate performance.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "Commit-hash deduplication and disjoint train/test splits (Table I) are good practices but are not leakage detection methods in the formal sense (no canary strings, membership inference, n-gram overlap analysis, or temporal splits).",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Identifier ingredients are prevalent: 85% of Defects4J fixes and 44% of TSSB-3M fixes require identifier ingredients.",
       "evidence": "Section III.C: 'in Defects4J only 15% of bugs involve no identifier ingredients' and 'in the former, 56% of the fixes do not require any identifier ingredient.'",
       "supported": "strong"
     },
     {
       "claim": "39-51% of fix identifier ingredients fall outside a typical 30-line context window.",
       "evidence": "Section III.C ingredient cover analysis: 'the input window's local context 61% and 49%... of ingredients for Defects4J and TSSB-3M, respectively.'",
       "supported": "strong"
     },
     {
       "claim": "ScanFix augmentation yields relative improvements of up to 31% over no-ingredient baselines for out-of-window bugs.",
       "evidence": "Table II: OOW scanner at t=0.05 achieves 24.56% vs 18.68% no-ingredients for winout bugs, a 31.5% relative improvement.",
       "supported": "strong"
     },
     {
       "claim": "A large-context baseline (5120 tokens) outperforms ScanFix despite requiring no domain-specific engineering.",
       "evidence": "Table II: Large Context achieves 27.60% for winout bugs vs 24.56% for best ScanFix variant; 47.8% relative improvement over no-ingredients.",
       "supported": "strong"
     },
     {
       "claim": "Perfect ingredient augmentation achieves 65.23% exact match for winout bugs, far exceeding all other conditions.",
       "evidence": "Table II: 'Perfect Ingrs.' achieves 65.23% for bugs with winout ingredients vs 27.60% for large context baseline.",
       "supported": "strong"
     },
     {
       "claim": "Repair success decreases as fix ingredient count increases and as ingredient distance from the bug location grows.",
       "evidence": "Figures 5, 6, and 7 consistently show downward repair success trends across ingredient count and distance, confirmed across multiple APR tools and both datasets.",
       "supported": "strong"
     },
     {
       "claim": "The scanner model achieves F1 of approximately 27% for fixall ingredients, with precision-recall tradeoffs controlled by threshold.",
       "evidence": "Section V.B: At t=0.5, recall=47.66% and precision=7.05%; paper reports F1 of 27% in abstract.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Identifier ingredients (variable/method/class names) are required in 44-85% of bug fixes, and 39-51% fall outside typical neural repair model context windows, causing significant repair failures. A dedicated StarEncoder-based scanner model can extract these ingredients from file-level context with modest F1 (~27%), and augmenting the repair model improves exact match by 7-31% relative over no-ingredient baselines. However, simply expanding the context window to 5120 tokens outperforms ScanFix by a larger margin, suggesting that domain-specific ingredient engineering may be rendered unnecessary by large-context models—a concrete instantiation of Sutton's bitter lesson. The gap between perfect-ingredient augmentation (65%) and ScanFix (25%) for out-of-window bugs shows the ceiling of the approach if extraction quality could be improved.",
   "red_flags": [
     {
       "flag": "Exact match proxy for repair",
       "detail": "TSSB-3M bugs are not executable, so exact match is used as the sole success metric. Exact match systematically underestimates real repair success and overestimates failure—a patch that is semantically equivalent but syntactically different counts as wrong. This affects all quantitative claims about improvement magnitude."
     },
     {
       "flag": "Single model architecture",
       "detail": "Both scanner and repair models use one architecture each (StarEncoder and CodeT5-small). The authors acknowledge this but note computational budget constraints. Results may not generalize to other model families."
     },
     {
       "flag": "No significance tests on main results",
       "detail": "Table II reports point estimates without significance tests. Given sample sizes in the hundreds of thousands, even small improvements may be significant, but the absence of tests leaves uncertainty about whether smaller differences (e.g., 'All' vs 'OOW' variants) are meaningful."
     },
     {
       "flag": "File-level scanning only for TSSB-3M",
       "detail": "The most valuable case—project-level scanning—cannot be evaluated on TSSB-3M due to scale. The paper acknowledges that 91-93% of fix ingredients exist at project level, but ScanFix is only evaluated on the file level where coverage is 69-77%. This artificially caps what the scanner can find."
     },
     {
       "flag": "Pre-training contamination unresolved",
       "detail": "The authors choose CodeT5-small to 'minimize' memorization issues but do not state pre-training cutoffs relative to TSSB-3M's GitHub data. The concern is acknowledged but not resolved."
     }
   ],
   "cited_papers": [
     {
       "title": "The plastic surgery hypothesis",
       "relevance": "Foundational work establishing that fix ingredients exist in project code; directly motivates the paper's research question about identifier ingredients in neural repair."
     },
     {
       "title": "Revisiting the Plastic Surgery Hypothesis via Large Language Models (FitRepair)",
       "relevance": "Most directly related prior work on using identifier ingredients with LLMs; ScanFix explicitly extends and compares against this approach."
     },
     {
       "title": "Out of context: How important is local context in neural program repair?",
       "relevance": "Direct predecessor by the same authors establishing context window size effects; ScanFix is the proposed solution to the problem identified there."
     },
     {
       "title": "TSSB-3M: Mining single statement bugs at massive scale",
       "relevance": "Primary dataset used for RQ1-4; the scale enables robust statistical analysis that Defects4J alone cannot provide."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "relevance": "Standard APR benchmark used for cross-tool analysis (RQ2) and the only dataset with full project-level context."
     },
     {
       "title": "RAP-Gen: Retrieval-Augmented Patch Generation with CodeT5 for Automatic Program Repair",
       "relevance": "Most related RAG-based alternative approach; the paper contrasts ScanFix's space-efficient identifier extraction with RAG's inclusion of full code fragments."
     },
     {
       "title": "Where were the repair ingredients for Defects4j bugs?",
       "relevance": "Prior empirical analysis of ingredient origin in Defects4J; this paper extends that work to NPR context and identifier-specific ingredients."
     },
     {
       "title": "Can OpenAI's codex fix bugs? an evaluation on QuixBugs",
       "relevance": "First-author prior work establishing memorization concerns in APR evaluation, directly cited to justify model size choice in this paper."
     },
     {
       "title": "Lost in the Middle: How Language Models Use Long Contexts",
       "relevance": "Cited to contextualize whether large-context baselines can effectively use all provided context, raising doubt about whether large windows fully solve the ingredient problem."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Research contribution to APR methodology; not directly usable as a practitioner tool, though the replication code is available."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The 'bitter lesson' framing—showing that simply increasing context window outperforms a more sophisticated approach—is mildly contrarian within the APR community."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code released on GitHub but requires training infrastructure and dataset preparation; not easily demoed."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Free University of Bozen-Bolzano and University of Bordeaux; not well-known AI labs."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "30665928",
         "title": "PERCEPT: Online change-point detection using topological data analysis",
         "points": 8,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=30665928"
       },
       {
         "hn_id": "42999205",
         "title": "Flip Graphs with Symmetry and New Matrix Multiplication Schemes",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=42999205"
       },
       {
         "hn_id": "44256016",
         "title": "Can Theoretical Physics Research Benefit from Language Agents?",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44256016"
       }
     ],
     "top_points": 8,
     "total_points": 12,
     "total_comments": 0
   }
 }