ai-research-survey

scan.json (31338B)

---

 {
   "paper": {
     "title": "Automatic Patch Correctness Assessment with Large Language Model",
     "authors": [
       "Xin Zhou",
       "Bowen Xu",
       "Kisub Kim",
       "DongGyun Han",
       "Hung Huu Nguyen",
       "Thanh Le-Cong",
       "Junda He",
       "Bach Le",
       "David Lo"
     ],
     "year": 2024,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2303.00202",
     "doi": "10.1109/TSE.2024.3452252"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM4PatchCorrect uses in-context learning with Starcoder-7B to assess patch correctness without needing labeled patches from the target APR tool, achieving 84.4% accuracy and 86.5% F1-score across 22 APR tools in cross-tool validation. The contrastive learning-based retrieval module for selecting semantically similar patches is the most impactful component, contributing 84.7% relative improvement in AUC. The approach significantly outperforms prior APCA methods (Cache, Quatrain, CodeBERT, ODS) on both Defects4J and Bears benchmarks, with all improvements statistically significant (Wilcoxon, p<0.05).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'We publicly share our implementation and dataset for future comparisons by the research community' in Section 6.5, but no repository URL or archive link is provided anywhere in the paper text."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available datasets: the merged dataset from Wang et al. [29] and Tian et al. [23] containing 1,179 patches from Defects4J [60], plus patches from the Bears benchmark [65]. These are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using a 2080-Ti GPU (12GB), HuggingFace library, and 8-bit quantization, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. The method is described algorithmically but there are no runnable commands or reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 2-9 report point estimates only (e.g., '84.4% accuracy'). No confidence intervals, error bars, or ± notation appears anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states: 'we conduct the Wilcoxon signed-rank tests between LLM4PatchCorrect and all baselines to investigate whether the improvements are significant. The results show that LLM4PatchCorrect is statistically significantly better than all baselines (all p-values are less than 0.05).'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are reported with baseline context throughout, e.g., '20.9% (84.4−69.8/69.8)' improvement over Tian et al.'s work in accuracy. Both absolute values and relative improvement ratios are provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 1,179 patches from 22 APR tools is used because it was available from prior work [23], [29]. No power analysis or justification for why this sample size is adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All tables show single-run point estimates. The contrastive learning training involves randomness but no multi-run analysis is presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: Patch-Sim [25], CodeBERT [40], Tian et al. [23], ODS [27], Quatrain [34], and Cache [33]. These span both dynamic and static APCA approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Cache (TOSEM 2022) and Quatrain (ASE 2022) represent the state-of-the-art at time of writing. CodeBERT is also a strong recent baseline. The baselines are competitive and appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 presents a systematic ablation study removing bug information, test information, and retrieved patches components individually and in combination, showing the contribution of each to overall performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Accuracy (Table 2), F1-score (Table 3), and AUC (Table 4). Both average and weighted average variants are reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the system's predictions is performed. The patch labels were created by developers in prior work, but the system's outputs are evaluated only by automated comparison against ground truth labels."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Cross-tool validation ensures strict separation: patches from the target APR tool form the test set, all other APR tools' patches form the training set. Hyper-parameter tuning uses a separate 5% split from the training data (Section 4.5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2-4 provide per-APR-tool breakdowns for all 22 APR tools, showing how performance varies across different target tools. Tables 8-9 show per-tool results for the Bears benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 provides a case study (Figure 10) showing how the model can make incorrect predictions without retrieved patches. The SOFix case with 1:10 class imbalance is discussed as a challenging scenario."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows several cases where LLM4PatchCorrect performs worse than baselines: CapGen (-2.4%), GenProg (-4.2%), Jaid (-1.9%), kPAR (-7.2%), TBar (-2.9%). These negative results are openly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '84.4% accuracy and 86.5% F1-score on average' match Table 2 and Table 3 averages. Claims of 10.2%-32.4% improvements over state-of-the-art are supported by the comparative results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) makes causal claims about component contributions using controlled single-variable manipulation: each component is added individually and in combination, providing adequate evidence for causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Automatic Patch Correctness Assessment with Large Language Model' implies general applicability, but all experiments use only Java patches from Defects4J and Bears benchmarks. The paper does not explicitly bound claims to Java or these specific benchmarks."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.5 discusses threats to validity (selection bias, prompt design, model under-training) but these are generic methodological limitations, not substantive alternative explanations for why the approach succeeds. For example, they never consider whether Starcoder's pre-training on Defects4J-related code could explain the performance gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures patch correctness classification (correct vs. overfitting) and frames results as patch correctness assessment. The measurement matches the claim — no proxy gap exists."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: Starcoder-7B [38], CodeBERT (0.13B) [40], CodeLlama-7B [46], CodeGen2-3.7B [67], Starcoder-3B, Starcoder-1B, BLOOM-1.7B. Table 1 catalogs all models with exact sizes."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompt template is provided in Section 3.1: '{test-patch} Q: It was wrong or correct? A: It was'. Figure 7 shows a complete example of the concatenated input including all guiding information components. Textual descriptions for each component are shown in Figure 2."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: k=10, β=0.9 (Section 4.5), contrastive learning training (lr=5e-5, batch size=64, epochs=3), CodeBERT fine-tuning (lr=1e-5, epochs=8), 8-bit quantization. The LLM is used for probability computation (argmax), not sampling."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a single-pass inference pipeline: retrieve similar patches, concatenate with guiding information, query the LLM for next-token probabilities."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents duplicate removal (string matching + manual semantic equivalence check, finding 2 pairs). The tokenization process (BPE) is described in Section 3.1. The patch embedding and retrieval pipeline is detailed in Section 3.2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 'Threats to Validity' provides substantive discussion organized into external validity, internal validity, and construct validity subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses study-specific threats: selection bias from considering only certain LLMs, dataset selection bias from using particular benchmarks, the manually crafted prompt may not be optimal, and the model may be under-trained."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. For example, it does not state that results are limited to Java patches, single-statement fixes, or the specific APR tools tested. The threats section discusses limitations but not explicit scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying patch datasets are publicly available through Wang et al. [29], Tian et al. [23], the Defects4J benchmark [60], and the Bears benchmark [65]. The patches and their correctness labels can be independently obtained."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the dataset: 1,179 patches from Defects4J merged from two existing large-scale datasets [23], [29], generated by 22 different APR tools, with labels 'carefully labeled and checked' by developers. Deduplication process is also described."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data consists of patches from standard software engineering benchmarks (Defects4J, Bears)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: datasets merged from [23] and [29], deduplication via string matching and manual review (Section 4.1), cross-tool validation setup (Section 4.2), hyper-parameter tuning on 5% held-out split (Section 4.5), and identical-patch removal between train/test to avoid data leakage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section appears in the paper text. Academic researchers at multiple institutions are almost certainly funded but no disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Singapore Management University, NC State, Royal Holloway University of London, and University of Melbourne. All are academic institutions with no apparent conflict with the evaluated tools."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding is disclosed, it is impossible to assess whether funders are independent of the outcome. The authors evaluate open-source models not produced by their institutions, suggesting likely independence, but this cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of Starcoder-7B's training data cutoff date. The paper cites the Starcoder paper [38] and notes it was trained on '1,000B tokens from 80+ programming languages' but does not state when the training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses removing identical patches between the cross-tool train/test splits (Section 4.2), but never addresses whether Starcoder's pre-training data might contain Defects4J patches or their correctness labels from online sources."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J was published in 2014 and its patches are widely discussed online. Starcoder was trained after 2014, so contamination is possible. The paper does not discuss this risk at all."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated using pre-labeled patch datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated patch correctness assessment using existing benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation is on pre-labeled patch correctness datasets."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard software engineering benchmarks."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions involving people."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or evaluators involved in the experimental evaluation."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 reports: 'LLM4PatchCorrect only costs 2.4 seconds for each patch' compared to Invalidator's five hours for dynamic features. The hardware requirement (2080-Ti GPU, 12GB) is also stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-patch inference time (2.4s) and GPU model (2080-Ti) are stated, but total compute budget for all experiments is not quantified. Training time for the contrastive learning model and total experiment duration are not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. The contrastive learning training involves randomness (dropout, batch sampling) but no seed sensitivity analysis is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs without explicit confirmation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 describes hyper-parameter tuning with specific ranges: β=[0.80, 0.82, 0.85, 0.87, 0.9, 0.92, 0.95] and k=[2, 4, 6, 8, 10, 12, 14, 15], totaling 56 configurations. Figure 8 shows the results."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 states hyper-parameter tuning was performed 'on a randomly split 5% of the labeled patch pools,' separate from the test data. The best configuration (k=10, β=0.9) was selected based on AUC on this held-out validation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Wilcoxon signed-rank tests are conducted between LLM4PatchCorrect and each of 5+ baselines, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper re-uses Cache's released implementation, which is good practice. However, for other baselines the paper does not discuss author-evaluation bias or the systematic tendency for authors' re-implementations of baselines to underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "LLM4PatchCorrect uses Starcoder-7B (7B parameters) while the CodeBERT baseline uses only 0.13B parameters — a ~54x size difference. This compute disparity is not discussed or controlled for in comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the Defects4J/Bears patch correctness benchmarks actually measure real-world patch assessment capability, or whether the binary correct/overfitting labels adequately capture patch quality."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. The approach is a direct inference pipeline without tool use or multi-step agent workflows."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Defects4J patches and their correctness labels have been available online since well before Starcoder's training, creating temporal leakage risk. The paper does not address whether the model might have encountered these patches during pre-training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper removes identical patches between train and test sets in cross-tool validation but does not discuss whether the guiding information (bug descriptions, execution traces, test cases, coverage) could leak correctness information to the model."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "In cross-tool validation, training and test patches often target the same bugs from different APR tools. Patches for the same bug share structural similarities (same buggy code, same tests). This non-independence is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes concrete deduplication: string matching for identical patches and manual examination for semantic equivalents, identifying 2 pairs of semantically equivalent patches which were removed. Section 4.2 also removes identical patches between train/test."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM4PatchCorrect achieves 84.4% accuracy and 86.5% F1-score on average across 22 APR tools in cross-tool validation without requiring labeled patches from the target APR tool.",
       "evidence": "Tables 2-3 show per-tool and average results. Average accuracy is 84.4% and average F1-score is 86.5% across 22 APR tools in leave-one-out cross-tool validation.",
       "supported": "strong"
     },
     {
       "claim": "LLM4PatchCorrect significantly outperforms all state-of-the-art APCA approaches, with improvements of 10.2% to 32.4% in accuracy, 6.1% to 24.1% in F1, and 10.1% to 33.2% in AUC over different baselines.",
       "evidence": "Tables 2-4 show comparisons against 6 baselines. Wilcoxon signed-rank tests confirm all improvements are statistically significant (p<0.05). Section 5.1 reports the improvement percentages.",
       "supported": "strong"
     },
     {
       "claim": "The contrastive learning-based patch retrieval module is the most impactful component, contributing 84.7% relative improvement in AUC over using the LLM alone.",
       "evidence": "Table 7 ablation study shows retrieval module improves accuracy by 11.4%, F1 by 3.9%, and AUC by 84.7% relative to the LLM alone.",
       "supported": "strong"
     },
     {
       "claim": "LLM4PatchCorrect generalizes beyond Defects4J to the Bears benchmark, outperforming the best baseline by 27.0% in accuracy and 25.8% in F1.",
       "evidence": "Tables 8-9 show results on Bears benchmark across 4 APR tools (Arja, GenProg, Kali, RSRepair). Average accuracy 92.1% vs Quatrain's 72.5%.",
       "supported": "moderate"
     },
     {
       "claim": "Larger LLM model sizes generally lead to better performance in the LLM4PatchCorrect framework.",
       "evidence": "Table 10 compares 6 LLMs from 1B to 7B parameters. 7B models (Starcoder-7B, CodeLlama-7B) generally outperform smaller variants, though Starcoder-1B shows competitive F1 (85.9%).",
       "supported": "moderate"
     },
     {
       "claim": "LLM4PatchCorrect can filter out 90.4% of overfitting patches while increasing the correct patch ratio from 21.7% to 63.7%.",
       "evidence": "Figure 9(b) confusion matrix shows 584/646 overfitting patches correctly identified, with 109 correct patches among remaining 171 predicted-correct patches.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance across runs",
       "detail": "All experimental results are reported as single point estimates without confidence intervals, standard deviations, or any uncertainty quantification. The contrastive learning training involves randomness, yet no multi-seed analysis is presented."
     },
     {
       "flag": "Pre-training contamination not addressed",
       "detail": "Starcoder-7B was pre-trained on massive code corpora. Defects4J patches and their correctness labels are widely available online and likely in the pre-training data. The paper never discusses whether the model might have memorized patch correctness information, which could inflate performance beyond what the in-context learning framework provides."
     },
     {
       "flag": "Unfair compute comparison with baselines",
       "detail": "LLM4PatchCorrect uses Starcoder-7B (7B parameters) plus a CodeBERT-based retrieval model, while the CodeBERT baseline uses only 0.13B parameters. This ~54x model size difference is not discussed or controlled for. The improvement might partly reflect the advantage of larger models rather than the proposed approach design."
     },
     {
       "flag": "Shared bugs between cross-tool train and test sets",
       "detail": "In cross-tool validation, patches in the training set and test set often target the same Defects4J bugs. A model could learn bug-specific patterns from training patches and apply them to test patches for the same bug, inflating performance beyond genuine cross-tool transfer."
     },
     {
       "flag": "No released code despite claiming public availability",
       "detail": "Section 6.5 states 'We publicly share our implementation and dataset for future comparisons' but no URL, repository link, or archive is provided in the paper."
     }
   ],
   "cited_papers": [
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "The primary LLM used in LLM4PatchCorrect; a major open-source code LLM trained on 1000B tokens from 80+ programming languages."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Used as the backbone for the contrastive learning patch embedding model and as a baseline for fine-tuning comparison."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "The Codex paper — foundational work on LLMs for code that established evaluation paradigms used in subsequent code LLM research."
     },
     {
       "title": "Code llama: Open foundation models for code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Alternative code LLM evaluated in the model comparison study (Table 10), showing comparable performance to Starcoder-7B."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["T. B. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "GPT-3 paper that introduced in-context learning, the core paradigm used by LLM4PatchCorrect for patch correctness assessment."
     },
     {
       "title": "LoRA: Low-rank adaptation of large language models",
       "authors": ["E. J. Hu", "Y. Shen", "P. Wallis"],
       "year": 2022,
       "arxiv_id": "2106.09685",
       "relevance": "Parameter-efficient fine-tuning method cited as context for why in-context learning (no fine-tuning) was chosen over full fine-tuning."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair — a related approach using pre-trained models for program repair without fine-tuning, contrasted with the authors' focus on patch correctness assessment."
     },
     {
       "title": "CodeT5: Identifier-aware unified pre-trained encoder-decoder models for code understanding and generation",
       "authors": ["Y. Wang", "W. Wang", "S. R. Joty"],
       "year": 2021,
       "arxiv_id": "2109.00859",
       "relevance": "A pre-trained encoder-decoder model for code, part of the landscape of code LLMs discussed in the background."
     },
     {
       "title": "Context-aware code change embedding for better patch correctness assessment",
       "authors": ["B. Lin", "S. Wang", "M. Wen", "X. Mao"],
       "year": 2022,
       "relevance": "Cache — the prior state-of-the-art in automatic patch correctness assessment, used as the primary baseline."
     },
     {
       "title": "SimCSE: Simple contrastive learning of sentence embeddings",
       "authors": ["T. Gao", "X. Yao", "D. Chen"],
       "year": 2021,
       "arxiv_id": "2104.08821",
       "relevance": "The contrastive learning framework adapted for patch embedding in LLM4PatchCorrect's retrieval module."
     },
     {
       "title": "Invalidator: Automated patch correctness assessment via semantic and syntactic reasoning",
       "authors": ["T. Le-Cong", "D.-M. Luong", "X. B. D. Le"],
       "year": 2023,
       "arxiv_id": "2301.01113",
       "relevance": "A hybrid dynamic+static APCA approach combining program invariants and code embeddings, compared against for efficiency (5 hours vs 2.4 seconds)."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners using APR tools could apply this to filter overfitting patches, reducing manual review effort."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Using LLMs for patch assessment is a natural extension of LLM-for-code trends; the cross-tool transfer finding is mildly surprising but not contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper is about improving software quality tooling."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; a straightforward technical contribution with incremental improvements over baselines."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No released code, demo, or installable tool despite claiming public availability."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses StarCoder (BigCode project) which has moderate recognition; authors from Singapore Management University have some SE research visibility."
     }
   }
 }