ai-research-survey

scan.json (20812B)

---

 {
   "paper": {
     "title": "ComPass: Contrastive Learning for Automated Patch Correctness Assessment in Program Repair",
     "authors": ["Quanjun Zhang", "Ye Shang", "Haichuan Hu", "Chunrong Fang", "Zhenyu Chen", "Liang Xiao"],
     "year": 2026,
     "venue": "Empirical Software Engineering",
     "arxiv_id": "2602.07561"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "An anonymous repository is provided: https://anonymous.4open.science/r/ComPass-7EF1/. The Data Availability Statement confirms code and dataset are available there."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset of 2,274 labeled patches from Defects4J is stated as available in the repository. Defects4J itself is a public benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 specifies: bert-base-uncased with 110M parameters from Hugging Face, PyTorch, Adam optimizer, two Tesla V100-SXM2 GPUs, Ubuntu 20.04. Specific library versions are not listed, but the environment is described in reasonable detail."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The repository link is provided but the paper does not describe how to run the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 88.35% accuracy) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ComPass 'significantly outperforms' baselines but provides no statistical significance tests (no p-values, no t-tests, no bootstrap tests). Comparisons are based solely on comparing percentage values."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are consistently reported, e.g., '6.33% higher than the second highest value obtained from APPT (i.e., 82.02%)'. Absolute values for both systems are given throughout."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset size of 2,274 patches is not justified. No power analysis or discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "5-fold cross-validation is used but no variance or standard deviation across folds is reported. Only aggregate metrics are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Seven baselines are included: PATCH-SIM, ODS, BERT (LR/DT/SVM), CACHE, APPT, and ChatGPT-3.5 (Section 4.3, Table 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "APPT (2024) is the most recent state-of-the-art baseline. ChatGPT-3.5 (gpt-3.5-turbo-0125) is also included as an LLM baseline. The baselines represent the current state of APCA research."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2.1 (Table 5) presents an ablation study removing contrastive learning, data augmentation, fine-tuning, and pre-training components individually. Section 5.2.2 (Table 6) ablates vector integration strategies."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: accuracy, precision, recall, and F1-score (Section 4.4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a binary classification task on patches with known ground-truth labels. Human evaluation of outputs is not relevant; the ground truth labels serve as the evaluation standard."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "5-fold cross-validation is used (Section 4.5), and a separate cross-project evaluation is conducted (Section 5.3) where entire projects are held out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 provides per-project breakdowns (Chart, Closure, Lang, Math, Time) in the cross-project evaluation setting."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and accompanying text analyze a specific failure case where ComPass incorrectly classifies an overfitting patch as correct, discussing why the semantic redundancy was missed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study shows performance drops when components are removed. The cross-project evaluation shows performance degradation compared to within-project (e.g., 88.35% → 81.01% for Chart). The paper also notes that removing cosine similarity actually increased precision by 1.47%."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 88.35% accuracy, 87.50% precision, 88.69% recall, 88.09% F1-score — all confirmed in Table 3. Claims about component contributions and generalizability are supported by Tables 5 and 7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'contrastive learning increases accuracy by 4.11%') are supported by controlled ablation studies where single components are removed (Table 5). This is adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java patches from Defects4J but the title and framing suggest general applicability to 'Program Repair' without language qualification. The threats section acknowledges Defects4J limitation but does not bound the title/abstract claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section discusses methodological limitations (BERT choice, Defects4J benchmark, baseline selection) but does not discuss alternative explanations for why ComPass outperforms baselines (e.g., could the improvement come from more training data via augmentation rather than contrastive learning specifically?)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 specifies 'bert-base-uncased' with 110M parameters from Hugging Face. The LLM baseline specifies 'gpt-3.5-turbo-0125'. These are sufficiently specific version identifiers."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the full prompt template used for the LLM-based baseline, including the exact text and expected output format."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5 reports: batch size 16, max input length 512, max epochs 50, learning rate 5e-5, hidden dimension 768, 12 encoder layers, 12 attention heads, Adam optimizer."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. ComPass is a standard pre-train then fine-tune pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes: patches collected from prior studies, duplicates removed by comparing code after removing whitespace/comments, unlabeled corpus constructed from earliest commits of Defects4J projects, deduplication against patch benchmark. Augmentation process described in Section 3.2 with 18 transformation rules (Table 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated, substantive discussion with four specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: (1) reliance on BERT specifically, (2) single benchmark Defects4J, (3) specific baseline selection, (4) evaluation setting focused on overfitting detection. Each includes concrete mitigation steps taken."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While threats are discussed, the paper does not explicitly state what it does NOT show. The threats note generalization uncertainty but do not explicitly bound claims — e.g., no statement that results are limited to Java, Defects4J, or the specific APR tools tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The Data Availability Statement states code and dataset are available at the anonymous repository. The patches are derived from the public Defects4J benchmark."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes collecting plausible patches from prior studies (Liu et al., Xiong et al., Ali et al., Tian et al., Lin et al.), filtering duplicates, and collecting 485,106 unlabeled code snippets from Defects4J earliest commits."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard benchmarks (Defects4J) and prior published patch sets."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: collect patches from prior work → remove duplicates → label inheritance from established benchmarks → code transformation for augmentation (1,234,020 augmented from 485,106 unlabeled; 10,983 from 2,274 labeled). Table 2 provides detailed per-tool statistics."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Declarations section states: 'This work is supported partially by the National Natural Science Foundation of China (61932012, 62141215, 62372228), Natural Science Foundation of Jiangsu Province (BK20251458) and Fundamental Research Funds for the Central Universities (AE89991/463).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Nanjing University of Science and Technology and Nanjing University. No evaluated product is affiliated with these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Chinese national science foundations and university funds, which have no financial stake in the outcome of patch correctness assessment research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": true,
         "justification": "The Declarations section explicitly states: 'The authors declared that they have no conflict of interest.'"
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "ComPass trains its own model (BERT fine-tuned with contrastive learning) rather than evaluating a pre-trained model's knowledge on a benchmark. The LLM baseline (GPT-3.5) is secondary and not the main contribution being evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper trains and evaluates its own model with explicit train/test splits via cross-validation, not evaluating pre-trained model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Not applicable for the same reason. The paper does address data leakage concerns for its own pre-training: 'we deduplicate all collected code snippets and the above patch benchmark' (Section 4.2)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper states 'Ethical approval: Not Applicable.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-patch processing time is reported, despite the paper positioning ComPass as a practical filtering component in APR pipelines."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned (two Tesla V100-SXM2 GPUs) but total training time, GPU hours, or computational budget are not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "ComPass achieves 88.35% accuracy, 87.50% precision, 88.69% recall, and 88.09% F1-score on 2,274 Defects4J patches.",
       "evidence": "Table 3 (Section 5.1) reports these exact numbers under 5-fold cross-validation.",
       "supported": "moderate"
     },
     {
       "claim": "ComPass significantly outperforms the state-of-the-art APPT by 6.33% accuracy.",
       "evidence": "Table 3 shows APPT at 82.02% accuracy vs ComPass at 88.35%. However, no statistical significance test is provided.",
       "supported": "moderate"
     },
     {
       "claim": "Contrastive learning increases accuracy by 4.11% and precision by 7.80%.",
       "evidence": "Table 5 (Section 5.2.1) ablation study: removing contrastive learning drops accuracy from 88.35% to 84.72% and precision from 87.50% to 80.67%.",
       "supported": "moderate"
     },
     {
       "claim": "ComPass is generalizable to existing learning-based APCA approaches and advanced PLMs.",
       "evidence": "Table 7 (Section 5.2.3) shows improvements of 4.92%-15.67% accuracy when integrating ComPass with Tian et al. variants, APPT, and CodeBERT.",
       "supported": "moderate"
     },
     {
       "claim": "ComPass achieves optimal performance in cross-project prediction scenarios.",
       "evidence": "Table 8 (Section 5.3) shows ComPass outperforms BERT SVM across all five Defects4J projects in cross-project setting, but only one baseline is compared.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ComPass introduces contrastive learning and data augmentation to automated patch correctness assessment, achieving 88.35% accuracy on 2,274 Defects4J patches, outperforming the prior state-of-the-art APPT by 6.33% accuracy. Ablation studies show all components (contrastive pre-training, data augmentation, fine-tuning) contribute positively. The framework generalizes across different encoder PLMs (BERT, CodeBERT) and existing APCA approaches, with 4.92%-15.67% accuracy improvements when integrated.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "The paper repeatedly claims ComPass 'significantly outperforms' baselines but provides no statistical tests. All comparisons are based on comparing point estimates from 5-fold cross-validation without reporting variance across folds."
     },
     {
       "flag": "No variance across folds reported",
       "detail": "Despite using 5-fold cross-validation, no standard deviation or variance across folds is reported for any experiment, making it impossible to assess result stability."
     },
     {
       "flag": "Cross-project evaluation uses only one baseline",
       "detail": "The cross-project evaluation (RQ3, Table 8) compares only against BERT SVM, omitting stronger baselines like APPT, weakening the claim of optimal cross-project performance."
     },
     {
       "flag": "Single benchmark, single language",
       "detail": "All experiments use only Java patches from Defects4J. Claims about general program repair applicability are not bounded to this setting."
     }
   ],
   "cited_papers": [
     {
       "title": "APPT: A pre-trained model-based approach for automated patch correctness assessment",
       "authors": ["Quanjun Zhang"],
       "year": 2024,
       "relevance": "Direct predecessor and primary baseline; PLM-based automated patch correctness assessment."
     },
     {
       "title": "Large language models for software engineering: Survey and open problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "relevance": "Survey of LLMs for software engineering tasks including program repair."
     },
     {
       "title": "CodeBERT: A pre-trained model for programming and natural languages",
       "authors": ["Zhangyin Feng", "Daya Guo"],
       "year": 2020,
       "relevance": "Foundation PLM for code understanding, used as alternative encoder in ComPass experiments."
     },
     {
       "title": "ContraBERT: Enhancing code pre-trained models via contrastive learning",
       "authors": ["Shangqing Liu", "Bozhi Wu"],
       "year": 2023,
       "relevance": "Contrastive learning applied to code PLMs for robustness improvement."
     },
     {
       "title": "Context-aware code change embedding for better patch correctness assessment",
       "authors": ["Bo Lin", "Shangwen Wang"],
       "year": 2022,
       "relevance": "CACHE: context-aware APCA technique used as baseline."
     },
     {
       "title": "Identifying test-suite-overfitted patches through test case generation",
       "authors": ["Xiong Yingfei"],
       "year": 2018,
       "relevance": "PATCH-SIM: dynamic-based APCA baseline and foundational work on patch overfitting."
     },
     {
       "title": "Evaluating representation learning of code changes for predicting patch correctness in program repair",
       "authors": ["Haoye Tian"],
       "year": 2020,
       "relevance": "Foundational APCA work using BERT representation learning, multiple baselines built on this."
     },
     {
       "title": "A survey on automated program repair techniques",
       "authors": ["Quanjun Zhang"],
       "year": 2023,
       "relevance": "Comprehensive APR survey covering the landscape of program repair techniques."
     },
     {
       "title": "Defects4J: A database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["Ren\u00e9 Just", "Darioush Jalali", "Michael D. Ernst"],
       "year": 2014,
       "relevance": "Standard benchmark for evaluating automated program repair and patch correctness assessment."
     },
     {
       "title": "INVALIDATOR: Automated patch correctness assessment via semantic and syntactic reasoning",
       "authors": ["Thanh Le-Cong"],
       "year": 2023,
       "relevance": "Recent APCA technique using program invariants and pre-trained models."
     }
   ]
 }