ai-research-survey

scan.json (30181B)

---

 {
   "paper": {
     "title": "Automatic Patch Correctness Assessment with Large Language Model",
     "authors": [
       "Xin Zhou",
       "Bowen Xu",
       "Kisub Kim",
       "DongGyun Han",
       "Hung Huu Nguyen",
       "Thanh Le-Cong",
       "Junda He",
       "Bach Le",
       "David Lo"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2303.00202",
     "doi": "10.48550/arXiv.2303.00202"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM4PatchCorrect uses Starcoder-7B with in-context learning to assess patch correctness in a cross-tool setting where no labeled patches from the target APR tool are available. It achieves 84.4% accuracy and 86.5% F1 on average across 22 APR tools, significantly outperforming prior static and dynamic APCA approaches. The contrastive learning-based retrieval of semantically similar patches is the most impactful component, contributing an 84.7% relative AUC improvement. Results generalize to the Bears benchmark with 92.1% average accuracy.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.5 states 'We publicly share our implementation and dataset for future comparisons' and Section 5.1 references a 'replication package,' but no URL, repository link, or archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available datasets from Wang et al. [29] and Tian et al. [23], merged by Lin et al. [33], containing 1,179 patches from the Defects4J benchmark [60]. The underlying data is publicly accessible through these prior works."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using a 2080-Ti GPU with 12GB memory, HuggingFace library, and int8 quantization, but provides no requirements.txt, Dockerfile, or detailed dependency listing with library versions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology conceptually but does not include a README, scripts, or commands to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2–4, 6, 8–10 report only point estimates for Accuracy, F1, and AUC. No confidence intervals, error bars, or ± notation appears anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states: 'we conduct the Wilcoxon signed-rank tests between LLM4PatchCorrect and all baselines to investigate whether the improvements are significant. The results show that LLM4PatchCorrect is statistically significantly better than all baselines (all p-values are less than 0.05).'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both absolute values and relative improvement percentages throughout, e.g., '20.9% (84.4−69.8/69.8)' improvement against Tian et al. in accuracy (Section 5.1), providing sufficient context for effect size interpretation."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 1,179 patches from 22 APR tools is used without any justification for adequacy or power analysis. Some individual APR tool test sets are very small (e.g., SOFix has only 11 patches, HDRepair 8 patches)."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results are single-point estimates with no indication of result stability."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are compared: Patch-Sim [25], CodeBERT [40], Tian et al. [23], ODS [27], Quatrain [34], and Cache [33], covering both dynamic and static APCA approaches."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Cache (Lin et al., 2022) and Quatrain (Tian et al., 2022) represent the state-of-the-art at the time of writing. The baselines span from 2018 (Patch-Sim) to 2022 (Cache), which is appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 (RQ2) presents a systematic ablation study in Table 7, evaluating the contribution of bug information, test information, and retrieved patches, building from LLM-only to the full model."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three evaluation metrics are used: Accuracy, F1-score, and AUC, reported in Tables 2, 3, and 4 respectively. Both averaged and weighted averaged results are provided."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using pre-existing correctness labels. No human evaluation of the system's predictions is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Cross-tool validation uses a leave-one-out design (Section 4.2): patches from each APR tool serve as the test set while patches from all other tools form the training set. Hyperparameter tuning used a separate 5% random split of the labeled patch pool (Section 4.5)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2–4 provide per-APR-tool breakdowns for all 22 tools, including the correct:wrong ratio for each. Tables 8–9 provide per-tool results on the Bears benchmark."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 provides a case study showing how the retrieval module corrects an initial misprediction. Section 5.1 discusses SOFix's low scores due to class imbalance (10:1 ratio) and ODS inference failures on certain patches."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2–4 show cases where LLM4PatchCorrect does not improve over baselines, e.g., CapGen (-2.4% accuracy), GenProg (-4.2%), kPAR (-7.2%), Jaid (-1.9%). These are reported transparently."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 84.4% accuracy and 86.5% F1, which match the average row in Tables 2 and 3. The claimed improvement ranges (10.2%–32.4% accuracy, 6.1%–24.1% F1, 10.1%–33.2% AUC) are supported by the tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 7) systematically adds components to isolate their individual contributions. The controlled single-variable manipulations in the ablation design support claims like 'the patch retrieval module leads to relative improvements of 11.4%, 3.9%, and 84.7%' (Section 5.2)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automatic Patch Correctness Assessment with Large Language Model' without bounding to Java. All experiments use only Java patches from Defects4J and Bears benchmarks. No explicit statement that results may not generalize to other languages or patch types."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 discusses specific threats: selection bias from choosing particular LLMs, dataset selection bias from using specific benchmarks, prompt design optimality ('we cannot ensure that our prompt is optimal'), and model under-training. These are specific to the study design."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures classification accuracy/F1/AUC on manually labeled patch correctness data, and claims to assess patch correctness. The measurement directly matches the claimed outcome — no proxy gap exists between the metric and the framing."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are provided: Starcoder-7B, Starcoder-1B, Starcoder-3B, CodeLlama-7B, CodeGen2-3.7B, BLOOM-1.7B, CodeBERT (Table 1, Table 10). These are versioned open-source models with fixed weights."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompt template is given in Section 3.1: '{test-patch} Q: It was wrong or correct? A: It was'. Guiding information templates are shown in Section 3.3 and Figure 2. Figure 7 shows a complete concatenated example input."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Contrastive learning: lr=5e-5, batch_size=64, epochs=3 (Section 3.2.3). CodeBERT fine-tuning: lr=1e-5, epochs=8 (Section 4.3). Retrieval: k=10, β=0.9 (Section 4.5). int8 quantization for inference (Section 3.4.2). Inference uses argmax probability comparison (deterministic)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a direct inference pipeline: retrieve similar patches → concatenate guiding information → single LLM inference pass."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents dataset merging from two sources, duplicate removal via string matching, manual semantic duplicate check (identifying 2 pairs of semantically equivalent patches), and BPE tokenization (Section 3.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.5 'Threats to Validity' contains three subsections: Threats to External Validity, Threats to Internal Validity, and Threats to Construct Validity, with substantive discussion in each."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: 'our study may have a selection bias by considering only several large pre-trained models,' 'the manually crafted prompt...we cannot ensure that our prompt is optimal,' and 'the large pre-trained model we employ...may have been under-trained.' These are specific to this study's design."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. No mention that results are limited to Java, to Defects4J-style bugs, or to the specific APR tool types tested. The future work section mentions other tasks but does not bound the current scope."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available patch datasets from Wang et al. [29] and Tian et al. [23], built on the public Defects4J [60] and Bears [65] benchmarks. Raw patches and labels are accessible through these prior works."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the dataset: 1,179 patches from 22 APR tools, merged from two large-scale datasets, with correctness labels 'carefully labeled and checked' by developers. Deduplication steps are documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (Defects4J, Bears) and previously published patch datasets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: two datasets merged (Section 4.1) → duplicate removal via string matching → manual semantic duplicate check (2 pairs found and removed) → cross-tool split for evaluation (Section 4.2) → tokenization via BPE (Section 3.1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant information appears anywhere in the paper text provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Singapore Management University, North Carolina State University, Royal Holloway University of London, and University of Melbourne."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence of funder from outcome cannot be verified."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interests declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Starcoder-7B's training data cutoff is not stated. The paper mentions it is pre-trained on '1,000 billion tokens from 80+ programming languages' but does not specify the temporal range of the training corpus."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "While Section 4.2 removes identical patches between the cross-tool train/test splits, there is no discussion of whether Starcoder-7B's pre-training data includes the Defects4J benchmark code, patches, or bug descriptions."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J has been publicly available since 2014 and its bugs/patches are widely discussed online. Starcoder was trained on public code repositories. The paper does not discuss whether Starcoder may have seen the benchmark data during pre-training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation uses automated classification on pre-labeled datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates patch classification on existing benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Related Work) states 'LLM4PatchCorrect only costs 2.4 seconds for each patch,' compared to Invalidator's 5 hours + 7 minutes per patch. Hardware is specified as a 2080-Ti GPU with 12GB memory."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions the 2080-Ti GPU and per-patch inference time (2.4s), but does not state total compute budget for the contrastive learning training, hyperparameter search, or full experimental evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. The contrastive learning training and any stochastic components are not analyzed for seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs without any indication of repetition."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports the hyperparameter search: β=[0.80, 0.82, 0.85, 0.87, 0.9, 0.92, 0.95] and k=[2, 4, 6, 8, 10, 12, 14, 15] (7×8=56 configurations), conducted on a randomly split 5% of the labeled patch pool."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 describes selection of k=10 and β=0.9 based on AUC performance on a separate 5% validation split of the labeled patch pool, with results visualized in Figure 8."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper performs Wilcoxon signed-rank tests across 22 APR tools for 6 baselines but does not mention any correction for multiple comparisons (e.g., Bonferroni, Holm)."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement most baselines themselves (except Cache, which uses the authors' released implementation). No discussion of self-comparison bias or the risk that their reimplementations may underperform the originals."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 10 compares different LLM sizes but does not report performance as a function of compute budget. The compute difference between LLM4PatchCorrect (Starcoder-7B inference + CodeBERT contrastive training) and baselines (e.g., CodeBERT fine-tuning) is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Defects4J or Bears benchmarks are valid proxies for real-world APCA scenarios. The paper does not question whether these benchmarks' patch distributions represent realistic deployment settings."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is a direct retrieval-augmented inference pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Defects4J bugs and patches were available online before Starcoder's training. The contrastive learning model (CodeBERT) was trained on ManySStuBs4J, but temporal ordering relative to evaluation data is not addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The guiding information (bug descriptions, execution traces, test cases, coverage) is provided as features, but there is no discussion of whether any of these features leak correctness information beyond what would be available in a realistic deployment scenario."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes duplicate removal via string matching and manual semantic duplicate checking (identifying 2 pairs). Section 4.2 removes identical patches between train/test splits 'to avoid the data leaking issue.' Different APR tools can generate identical patches for the same bug."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Concrete deduplication methods are applied: string matching for identical patches and manual examination for semantic equivalence (Section 4.1). Section 4.2 explicitly removes training patches identical to test patches."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM4PatchCorrect achieves 84.4% accuracy and 86.5% F1-score on average across 22 APR tools in cross-tool validation.",
       "evidence": "Tables 2 and 3 (Section 5.1) show per-tool and averaged results. Wilcoxon signed-rank tests confirm statistical significance (all p < 0.05).",
       "supported": "strong"
     },
     {
       "claim": "LLM4PatchCorrect outperforms all baseline APCA approaches including Cache (state-of-the-art) by 14.7% accuracy, 6.8% F1, and 30.7% AUC on average.",
       "evidence": "Tables 2–4 provide head-to-head comparisons. Improvements over Cache: 84.4 vs 73.6 accuracy, 86.5 vs 81.0 F1, 80.4 vs 61.5 AUC (Section 5.1).",
       "supported": "strong"
     },
     {
       "claim": "The contrastive learning-based patch retrieval module is the most impactful component, contributing 84.7% relative AUC improvement.",
       "evidence": "Table 7 (Section 5.2) shows ablation results: AUC jumps from 41.8 (LLM only) to 77.2 (LLM + retrieved patches). Bug/test information adds smaller incremental gains.",
       "supported": "strong"
     },
     {
       "claim": "LLM4PatchCorrect generalizes to the Bears benchmark with 92.1% average accuracy, outperforming the best baseline by 27.0%.",
       "evidence": "Tables 8–9 (Section 6.2) show results on Bears across 4 APR tools. Best baseline Quatrain achieves 72.5% vs LLM4PatchCorrect's 92.1%.",
       "supported": "moderate"
     },
     {
       "claim": "LLM4PatchCorrect can filter out 90.4% of overfitting patches while increasing the correct patch ratio from 21.7% to 63.7%.",
       "evidence": "Confusion matrix analysis in Section 5.1 and Figure 9(b): 584/646 overfitting patches correctly identified, remaining patches have 109/(109+62) = 63.7% correct ratio.",
       "supported": "strong"
     },
     {
       "claim": "Larger LLM model sizes generally yield better performance within the LLM4PatchCorrect framework.",
       "evidence": "Table 10 (Section 6.3) compares 6 models from 1B to 7B parameters. Starcoder-7B leads in accuracy (84.4%) and F1 (86.5%), though Starcoder-1B is competitive in F1 (85.9%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or seed sensitivity reporting",
       "detail": "All results are single-point estimates with no standard deviations, confidence intervals, or multi-seed analysis. The contrastive learning component involves stochastic training, but stability is never assessed."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "Starcoder-7B was trained on public code repositories that likely include Defects4J patches and bug-related discussions. The paper never discusses whether the LLM may have memorized correctness patterns for these specific bugs."
     },
     {
       "flag": "Promised artifacts not provided",
       "detail": "Section 6.5 claims 'We publicly share our implementation and dataset' and Section 5.1 references a 'replication package,' but no URL, repository link, or archive identifier is provided in the paper."
     },
     {
       "flag": "Very small test sets for some APR tools",
       "detail": "Several APR tools have tiny test sets: SOFix (11 patches, 10:1 ratio), HDRepair (8 patches), Cardumen (9 patches, 0 correct). Results on these tools have high variance and limited statistical meaning."
     },
     {
       "flag": "Bears benchmark evaluation limited",
       "detail": "The Bears benchmark evaluation (Section 6.2) uses only 4 APR tools, and some have very few patches (Kali: 6, RSRepair: 2). The 100% accuracy on RSRepair with only 2 patches is not meaningful."
     }
   ],
   "cited_papers": [
     {
       "title": "Starcoder: may the source be with you!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Core LLM backbone used in this work; key open-source code LLM for evaluating in-context learning on SE tasks."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang"],
       "year": 2020,
       "arxiv_id": "2002.08155",
       "relevance": "Used as baseline and as the base model for contrastive learning patch embeddings; foundational code representation model."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["B. Roziere", "J. Gehring", "F. Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Alternative open-source code LLM compared in the model size analysis experiment."
     },
     {
       "title": "Language Models are Few-Shot Learners",
       "authors": ["T. B. Brown", "B. Mann", "N. Ryder"],
       "year": 2020,
       "arxiv_id": "2005.14165",
       "relevance": "Introduced in-context learning paradigm that LLM4PatchCorrect builds upon."
     },
     {
       "title": "Context-aware code change embedding for better patch correctness assessment",
       "authors": ["B. Lin", "S. Wang", "M. Wen", "X. Mao"],
       "year": 2022,
       "relevance": "State-of-the-art APCA baseline (Cache) that this work outperforms; provides the evaluation dataset."
     },
     {
       "title": "Evaluating representation learning of code changes for predicting patch correctness in program repair",
       "authors": ["H. Tian", "K. Liu", "A. K. Kaboré"],
       "year": 2020,
       "relevance": "Prior APCA approach using code representation learning as baseline; provides part of the evaluation dataset."
     },
     {
       "title": "Invalidator: Automated patch correctness assessment via semantic and syntactic reasoning",
       "authors": ["T. Le-Cong", "D.-M. Luong", "X. B. D. Le"],
       "year": 2023,
       "arxiv_id": "2301.01113",
       "relevance": "Related APCA approach combining dynamic and static features; compared on inference time (5 hours vs 2.4 seconds)."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair uses infilling-style LLM for program repair without fine-tuning, related zero-shot approach in APR domain."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex/HumanEval paper, foundational work on LLMs for code that motivates using LLMs for SE tasks."
     },
     {
       "title": "Automated patch correctness assessment: How far are we?",
       "authors": ["S. Wang", "M. Wen", "B. Lin"],
       "year": 2020,
       "relevance": "Large-scale empirical study of APCA approaches; provides evaluation dataset and Patch-Sim comparison data."
     },
     {
       "title": "Automated classification of overfitting patches with statically extracted code features",
       "authors": ["H. Ye", "J. Gu", "M. Martinez"],
       "year": 2022,
       "relevance": "ODS baseline using AST-level static code features for patch correctness classification."
     },
     {
       "title": "SimCSE: Simple contrastive learning of sentence embeddings",
       "authors": ["T. Gao", "X. Yao", "D. Chen"],
       "year": 2021,
       "arxiv_id": "2104.08821",
       "relevance": "Contrastive learning method (dropout-based positive sampling) adapted for patch embedding in this work."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Directly useful for APR practitioners who need to filter overfitting patches, but requires implementing the full pipeline with Starcoder-7B and contrastive learning."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Applying LLMs via in-context learning to patch assessment is a natural extension rather than a surprising finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the work is about improving software quality assurance."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts with existing work; straightforward performance comparison."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository URL, demo, or installable tool is provided despite claims of sharing artifacts."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses Starcoder (BigCode project) and CodeBERT — known models but not top-tier brand recognition like GPT-4 or Copilot."
     }
   }
 }