ai-research-survey

scan.json (31809B)

---

 {
   "paper": {
     "title": "Invalidator: Automated Patch Correctness Assessment via Semantic and Syntactic Reasoning",
     "authors": [
       "Thanh Le-Cong",
       "Duc-Minh Luong",
       "Xuan Bach D. Le",
       "David Lo",
       "Nhat-Hoa Tran",
       "Bui Quang-Huy",
       "Quyet-Thang Huynh"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2301.01113",
     "doi": "10.1109/TSE.2023.3255177"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "INVALIDATOR combines semantic reasoning (program invariants via Daikon) and syntactic reasoning (CodeBERT embeddings + Logistic Regression) to detect overfitting patches in automated program repair, achieving 81% accuracy and 0.87 F1-score on 139 patches from Defects4J. It outperforms seven baselines by 14-19% on accuracy/F1, correctly classifying 79% of overfitting patches with 97% precision. An ablation study shows both components contribute: the semantic classifier detects 51% of overfitting patches threshold-free, while the syntactic classifier provides broader coverage. The three best techniques (INVALIDATOR, ODS, RGT) are complementary, together covering 107 of 109 overfitting patches.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 9 states: 'INVALIDATOR is publicly available at https://github.com/thanhlecongg/Invalidator' and 'All materials including implementation, datasets, and experimental results are also published via https://doi.org/10.5281/zenodo.7699142.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is from publicly available prior work (Defects4J, Xiong et al., Wang et al.), and the authors release all materials including datasets via the Zenodo archive (doi:10.5281/zenodo.7699142)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Python programming language' and 'HuggingFace's Transformers framework' (Section 5.1.3) but provides no version numbers, requirements.txt, Dockerfile, or detailed dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository URL and Zenodo archive with implementation, datasets, and experimental results. The methodology is described in detail sufficient for replication, and the public repository is expected to contain runnable code."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-7 are reported as point estimates (e.g., Accuracy 0.81, F1-score 0.87) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims INVALIDATOR 'outperforms' and 'substantially outperforms' baselines based solely on comparing raw metric values (e.g., 0.81 vs. 0.68 accuracy). No statistical significance tests (t-test, bootstrap, etc.) are reported anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both absolute scores and relative improvements with baseline context, e.g., '14% (0.81 vs. 0.68)' for Accuracy and '19% (0.87 vs. 0.76)' for F1-score (Section 5.3.1), providing sufficient context for effect size interpretation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation set contains only 139 patches (109 overfitting, 30 correct). No power analysis or justification for why this sample size is sufficient for the claims made. The class imbalance is also not formally addressed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. The syntactic classifier involves ML training (Logistic Regression) that could vary across random seeds, but only single-run results are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven baselines are compared: RGT, ODS, BERT+LR, PATCHSIM, DIFFTGEN, ANTI-PATTERNS, and GT-INVARIANT (Section 5.2, RQ1). Results for all baselines are presented in Table 3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include ODS (2021), RGT (2021), BERT+LR (2020), and PatchZero (2023 in related work). These were state-of-the-art at time of writing. Older baselines (ANTI-PATTERNS 2016, DIFFTGEN 2017) are also included for completeness."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 5.3.4) conducts a thorough ablation: removing syntactic classifier (Table 6), removing semantic classifier (Table 6), comparing invariant granularities (Table 7), and evaluating individual overfitting rules (Figure 9)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five evaluation metrics are used: Recall, Precision, Accuracy, F1-score, and AUC (Section 5.1.2). Results are reported across all metrics in Tables 3-5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated against pre-labeled ground truth patches. No human evaluation of INVALIDATOR's outputs is performed. The ground truth labels were created by prior work, not as part of this study's evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses Xiong et al.'s 139 patches as a held-out test set, while training/validation uses Wang et al.'s data + Defects4J patches. Duplicates were removed: 'we removed a patch if it is syntactically equivalent to a patch in the evaluation set' (Section 5.1.1). Threshold tuned on validation set, not evaluation set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by technique comparison (Table 3), by classifier component (Table 6), by invariant granularity (Table 7), and by overfitting rule (Figure 9). A Venn diagram shows complementarity of top techniques (Figure 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses 3 false positives (correct patches misclassified as overfitting) and analyzes case studies of unique detections (Section 5.3.1, Math-58 example in Figure 6). Time efficiency limitations of invariant inference are also discussed (Section 6.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that INVALIDATOR slightly underperforms BERT+LR and PATCHSIM on Precision (Section 5.3.1), that the semantic classifier alone has limited recall (51%), and that invariant inference is 'time-consuming' with a 5-hour limit per patch (Section 6.1)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 79% overfitting detection, 23% improvement over best baseline recall, and 14%/19% improvements in Accuracy/F1 are all directly supported by Table 3 results (86/109 TP, 0.81 vs 0.68 accuracy, 0.87 vs 0.76 F1)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via ablation studies (RQ4): 'removing INVALIDATORsyn leads to a decrease of 26% and 23%' and 'without INVALIDATORsem, INVALIDATOR's performance also drops by 11% and 8%.' The ablation design uses controlled single-variable manipulation (removing one component at a time), which is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Automated Patch Correctness Assessment' and abstract claim broad applicability, but results are only on Java programs from 4 Defects4J projects (Chart, Time, Lang, Math). While the threats-to-validity section notes the Defects4J limitation, the title and abstract do not bound claims to Java or this specific benchmark."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses specific alternative explanations: ground truth labels may have 'subjective bias' from human annotators (construct validity), the 21 APR techniques may not represent all approaches (external validity), and baseline threshold tuning on evaluation sets may inflate their precision (Section 5.3.1)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures patch correctness classification against human-labeled ground truth. The measurement (TP/FP/TN/FN against labeled patches) matches the claim (automated patch correctness assessment). No proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "CodeBERT is specified by name with reference to Feng et al. (2020) and the official GitHub repository. CodeBERT has a single public release, making the name sufficiently specific. Daikon is also identified as the invariant inference tool with documentation reference."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. CodeBERT is used as a feature extractor (embedding generation), not as a prompted generative model. Daikon is a dynamic analysis tool, not prompt-based."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "The classification threshold T=0.975 is reported (Section 5.1.3), with extensive sensitivity analysis across the full range (0, 1) in RQ3. The embedding dimension k=768 and the 5-hour invariant inference time limit are stated. The approach uses CodeBERT for feature extraction (no fine-tuning) and standard Logistic Regression."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. INVALIDATOR is a two-phase classification pipeline (semantic invariant checking followed by syntactic ML classification), not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 documents the full pipeline: 220 patches from Xiong et al. + 902 from Wang et al. → filtered to 4 Defects4J projects (139 + 666) → added 223 developer patches → removed duplicates via syntactic equivalence → 746 train/val + 139 eval → 90/10 train/val split (671/75)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 'Threats to validity' contains three substantive subsections: external validity, internal validity, and construct validity, each discussing specific concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: '885 patches generated from 21 popular APR techniques. This may not represent all APR techniques,' 'patches in our dataset are only generated for the Defects4J dataset,' and 'correctness of the patches may be subject to subjective bias because they were manually labeled by human annotators' (Section 6.3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states what was not tested: only Defects4J (not Bears or Bugs.jar), only Java programs, only 4 projects. Section 6.2 acknowledges 'the reliance on ground truth patches limits our applications on pure APR problem settings.' QuixBugs was excluded with explanation."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All materials including datasets and experimental results are published via Zenodo (doi:10.5281/zenodo.7699142). The underlying patch datasets from Xiong et al. and Wang et al. are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 describes data collection: 220 patches from Xiong et al. [28] and 902 from Wang et al. [50], filtered to 4 Defects4J projects, supplemented with 223 developer-written patches from Defects4J. Inclusion/exclusion criteria and deduplication are described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard published benchmark datasets (Defects4J patches from prior work)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 traces the full pipeline with counts at each stage: 220+902 initial → 139+666 after project filtering → +223 developer patches = 1028 → remove duplicates = 746 for train/val, 139 for eval → 671 train + 75 validation. Table 1 and Table 2 summarize the splits."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Melbourne, Hanoi University of Science and Technology, and Singapore Management University. Email addresses are provided. The authors are not affiliated with any APR tool vendor being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence because no funding is disclosed. The authors are academic researchers not evaluating their own commercial product, but without funding disclosure this cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "CodeBERT is used as a feature extractor but its training data cutoff date is not stated. CodeBERT was pretrained on GitHub code which could include Defects4J project source code."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses train/test overlap for their own classifier: 'we removed a patch if it is syntactically equivalent to a patch in the evaluation set' (Section 5.1.1). However, potential overlap between CodeBERT's pretraining data and Defects4J code is not discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeBERT's pretraining corpus (GitHub code) contains Defects4J source code, which could influence the learned code representations used for feature extraction."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a benchmark evaluation of automated patch correctness assessment techniques."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates machine-generated patches against existing labeled datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 reports: 'assessing the correctness of 139 patches in our evaluation dataset... INVALIDATOR took 15.5 hours (i.e., about 7 minutes for each patch).' A 5-hour time limit for invariant inference per patch is also stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No hardware specifications (GPU/CPU type), total GPU hours, or training time for the Logistic Regression model are reported. Only inference wall-clock time is mentioned."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or sensitivity analysis across seeds. The Logistic Regression training involves randomness but only single-run results are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from a single run with no indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The threshold T=0.975 was tuned on the validation set, but no details on the search procedure (how many values tried, search method) are provided. Sensitivity analysis in RQ3 explores the full range but this is post-hoc analysis, not the search budget."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The classification threshold T=0.975 was 'yielded the highest precision on the validation dataset' (Section 5.1.3). Importantly, tuning was done on an independent validation set rather than the evaluation set, avoiding overfitting. RQ3 provides extensive threshold sensitivity analysis."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares against 7 baselines and makes multiple performance claims, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented INVALIDATOR and compare it against baselines (some re-implemented, some from prior work results). The bias of evaluating their own system is never acknowledged. For DIFFTGEN and GT-INVARIANT, they ran external implementations, but for their own system no independent evaluation is performed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "INVALIDATOR requires up to 5 hours of invariant inference per patch, which is substantially more compute than most baselines. This compute difference is not discussed in terms of performance-per-compute comparisons with baselines."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the Defects4J patch correctness labels actually measure real-world patch assessment ability. The construct validity of using manually labeled patches from specific Java projects as a proxy for general patch correctness assessment is not questioned."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. INVALIDATOR is a classification pipeline, not an agentic system with scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CodeBERT's pretraining data (collected before 2020) includes Defects4J source code or patches created before that date. The temporal relationship between training data and evaluation data is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the feature extraction pipeline (CodeBERT embeddings of buggy, patched, and correct code) could leak information about patch correctness through unintended channels."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1.1 explicitly addresses non-independence: 'there may be duplication between Wang et al.'s dataset, Defects4J's patches, and Xiong et al.'s dataset. To avoid data leakage, we removed the duplicated patches from the training and validation set.'"
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A concrete decontamination method is applied: 'we removed a patch if it is syntactically equivalent to a patch in the evaluation set' (Section 5.1.1), reducing the training set from 889 to 746 patches."
       }
     }
   },
   "claims": [
     {
       "claim": "INVALIDATOR correctly classified 79% of overfitting patches (86/109) with 97% precision on the evaluation dataset.",
       "evidence": "Table 3 shows 86 TP, 23 FN, 3 FP, 27 TN, yielding Recall=0.79, Precision=0.97, Accuracy=0.81, F1=0.87 on 139 patches (Section 5.3.1).",
       "supported": "strong"
     },
     {
       "claim": "INVALIDATOR outperforms the best baselines (ODS and RGT) by 19% in Accuracy and 14% in F1-score.",
       "evidence": "Table 3: INVALIDATOR achieves 0.81 Accuracy and 0.87 F1 vs. ODS/RGT at 0.68 Accuracy and 0.76 F1 (Section 5.3.1).",
       "supported": "moderate"
     },
     {
       "claim": "The semantic-based classifier alone detects 51% of overfitting patches with 97% precision.",
       "evidence": "Table 6 ablation: INVALIDATORsem alone achieves 56 TP out of 109 overfitting patches with 2 FP, giving Recall=0.51, Precision=0.97 (Section 5.3.4, RQ4.1).",
       "supported": "strong"
     },
     {
       "claim": "Syntactic reasoning boosts Accuracy by 35% and F1 by 30% over the semantic classifier alone.",
       "evidence": "Table 6: INVALIDATOR (0.81 Accuracy, 0.87 F1) vs. semantic-only (0.60 Accuracy, 0.67 F1). Section 5.3.4, RQ4.1.",
       "supported": "moderate"
     },
     {
       "claim": "INVALIDATOR is the most effective and stable technique among threshold-dependent APAC approaches.",
       "evidence": "Figure 8 shows INVALIDATOR maintains F1 above 0.8 across thresholds 0.1-0.9, while ODS and BERT+LR degrade significantly above 0.4 (Section 5.3.3, RQ3.2).",
       "supported": "moderate"
     },
     {
       "claim": "Using invariants from all executed methods improves performance over buggy-methods-only invariants by 28% Accuracy and 31% F1.",
       "evidence": "Table 7: executed methods yield 0.60 Accuracy and 0.67 F1 vs. buggy methods at 0.47 Accuracy and 0.51 F1 (Section 5.3.4, RQ4.2).",
       "supported": "strong"
     },
     {
       "claim": "INVALIDATOR, ODS, and RGT are complementary — together covering 107/109 overfitting patches.",
       "evidence": "Figure 5 Venn diagram shows their union covers 107/109 overfitting patches, with each detecting unique patches (Section 5.3.1).",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All performance comparisons are based on raw metric differences without any statistical testing (no p-values, confidence intervals, or bootstrap tests). Claims of 'outperforming' baselines rest entirely on comparing point estimates from a single run on 139 patches."
     },
     {
       "flag": "Small evaluation set",
       "detail": "The evaluation set contains only 139 patches (109 overfitting, 30 correct). With such a small sample, especially only 30 correct patches, the precision and accuracy estimates have high uncertainty that is never quantified."
     },
     {
       "flag": "Requires ground truth patches",
       "detail": "INVALIDATOR requires developer-written correct patches as input, which significantly limits practical applicability. This is acknowledged in Section 6.2 but the abstract and title do not make this fundamental limitation prominent."
     },
     {
       "flag": "No random seed analysis",
       "detail": "The ML-based syntactic classifier (Logistic Regression on CodeBERT features) involves training randomness, but results are from a single run with no variance across seeds reported."
     },
     {
       "flag": "Potential baseline implementation advantage",
       "detail": "Some baseline results are collected from prior work while INVALIDATOR is implemented and evaluated by its own authors. The bias of self-evaluation (Lucic et al. 2018) is not acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Genprog: A generic method for automatic software repair",
       "authors": ["C. Le Goues", "T. Nguyen", "S. Forrest", "W. Weimer"],
       "year": 2011,
       "relevance": "Foundational automated program repair technique using genetic programming, shown to produce 98% overfitting patches."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang", "N. Duan", "X. Feng", "M. Gong", "L. Shou", "B. Qin", "T. Liu", "D. Jiang", "M. Zhou"],
       "year": 2020,
       "relevance": "Pre-trained code representation model used as the feature extractor in INVALIDATOR's syntactic classifier."
     },
     {
       "title": "The daikon system for dynamic detection of likely invariants",
       "authors": ["M. D. Ernst", "J. H. Perkins", "P. J. Guo", "S. McCamant", "C. Pacheco", "M. S. Tschantz", "C. Xiao"],
       "year": 2007,
       "relevance": "Dynamic invariant inference tool central to INVALIDATOR's semantic-based patch classification approach."
     },
     {
       "title": "Sequencer: Sequence-to-sequence learning for end-to-end program repair",
       "authors": ["Z. Chen", "S. J. Kommrusch", "M. Tufano", "L.-N. Pouchet", "D. Poshyvanyk", "M. Monperrus"],
       "year": 2019,
       "relevance": "Deep learning approach to automated program repair using sequence-to-sequence models."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM-based approach to automated program repair, relevant to understanding LLM capabilities in code generation."
     },
     {
       "title": "Evaluating representation learning of code changes for predicting patch correctness in program repair",
       "authors": ["H. Tian", "K. Liu", "A. K. Kaboré", "A. Koyuncu", "L. Li", "J. Klein", "T. F. Bissyandé"],
       "year": 2020,
       "relevance": "BERT+LR baseline technique using pre-trained language model representations for patch correctness prediction."
     },
     {
       "title": "Defects4j: A database of existing faults to enable controlled testing studies for java programs",
       "authors": ["R. Just", "D. Jalali", "M. D. Ernst"],
       "year": 2014,
       "relevance": "Standard benchmark dataset for automated program repair evaluation, used as the basis for all experiments."
     },
     {
       "title": "Sapfix: Automated end-to-end repair at scale",
       "authors": ["A. Marginean", "J. Bader", "S. Chandra", "M. Harman", "Y. Jia", "K. Mao", "A. Mols", "A. Scott"],
       "year": 2019,
       "relevance": "Facebook's industrial-scale automated bug-fixing system, demonstrating practical APR deployment."
     },
     {
       "title": "PatchZero: Zero-shot automatic patch correctness assessment",
       "authors": ["X. Zhou", "B. Xu", "K. Kim", "D. Han", "T. Le-Cong", "J. He", "B. Le", "D. Lo"],
       "year": 2023,
       "arxiv_id": "2303.00202",
       "relevance": "Zero-shot approach to patch correctness assessment using LLMs, extending the APCA paradigm without labeled data."
     },
     {
       "title": "Neural program repair with execution-based backpropagation",
       "authors": ["H. Ye", "M. Martinez", "M. Monperrus"],
       "year": 2022,
       "relevance": "RewardRepair technique integrating execution feedback into neural program repair training, relevant to learning-based APR."
     },
     {
       "title": "Automated classification of overfitting patches with statically extracted code features",
       "authors": ["H. Ye", "J. Gu", "M. Martinez", "T. Durieux", "M. Monperrus"],
       "year": 2021,
       "relevance": "ODS baseline using 4199 hand-crafted code features for overfitting patch classification."
     },
     {
       "title": "Automated patch assessment for program repair at scale",
       "authors": ["H. Ye", "M. Martinez", "M. Monperrus"],
       "year": 2021,
       "relevance": "RGT baseline using random testing with ground truth for automated patch correctness assessment."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides a usable tool for APR researchers to filter overfitting patches, but requires ground truth patches which limits practical deployment."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Combining semantic and syntactic reasoning is a novel approach but the finding that invariants detect overfitting is not surprising given prior work by Yang and Yang (2020)."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns; focuses on software engineering quality assurance."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; straightforward technical contribution."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code and data publicly released on GitHub and Zenodo, reproducible by researchers with Java/Defects4J infrastructure."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Published in IEEE TSE (top SE journal) with David Lo (well-known SE researcher) as co-author, but not from a major AI lab."
     }
   }
 }