ai-research-survey

scan.json (20297B)

---

 {
   "paper": {
     "title": "ContrastRepair: Enhancing Conversation-Based Automated Program Repair via Contrastive Test Case Pairs",
     "authors": ["Jiaolong Kong", "Mingfei Cheng", "Xiaofei Xie", "Shangqing Liu", "Xiaoning Du", "Qi Guo"],
     "year": 2024,
     "arxiv_id": "2403.01971"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions patches are open-sourced for public evaluation (Section 5.2), but no repository URL for the ContrastRepair tool itself is provided in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmark datasets: Defects4j, QuixBugs, and HumanEval-Java, all of which are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions using Python lib-javaobj and Java lib-Javassist but does not provide version details or a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README with commands, or scripts for replicating experiments are provided in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Results are presented as point estimates (e.g., 103 bugs fixed)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ContrastRepair 'significantly outperforms' baselines but provides no statistical significance tests (no p-values, t-tests, etc.)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '15.32% more than the best baseline' and specific counts like '103 out of 255 bugs' vs '90' for CHATREPAIR."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes used. The Defects4j 2.0 subset of 82 bugs is noted as due to 'budget constraint' but no power analysis or formal justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "In the ablation study (RQ4, Table 8), the experiments are repeated three times and average results are reported. However, no standard deviation or spread measure is given, only averages. But the paper does report results across multiple runs to mitigate randomness."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against seven baselines: SelfAPR, AlphaRepair, RewardRepair, Recoder, CURE, TBar, and CHATREPAIR, plus a BaseChatGPT baseline (Section 4.1.4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include CHATREPAIR (2023), AlphaRepair (2022), SelfAPR (2022), and other recent methods. These represent the state of the art at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 4.5) presents ablation study with w/o Pair, w/o Similarity, and w/o Context variants. Table 8 and Table 9 show results."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses #Correct (correct fixes), #Plausible (plausible fixes), and #Query (API call efficiency) as metrics (Section 4.1.5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three SE researchers manually verified whether plausible patches were semantically correct, discussed disagreements, and reached consensus (Section 5.2, threats to validity)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "HumanEval-Java is used as an unseen dataset (RQ2) to address data leakage concerns, as it postdates GPT-3.5's training data cutoff."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-project breakdowns (Chart, Closure, Lang, Math, Mockito, Time). Table 3 shows per-scenario breakdowns (SL, SH, SF)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where type-aware mutation cannot generate passing tests (functions with no primitive-type parameters), and discusses limitations of the approach for logical bugs (Section 3.2.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the 12x10 configuration was least effective (Table 7), and that misleading test pairs can hurt repair (motivation example, Fig 1b). The w/o Context ablation shows decreased performance."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '143 out of all 337 bug cases' on Defects4j 1.2 and 2.0 combined (103+40=143 from Tables 1), and the best baseline fixes 124 (90+34=124). These match the results tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about contrastive pairs improving repair ('ContrastRepair improves...') and supports these with ablation studies (w/o Pair, w/o Similarity, w/o Context) that use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'new state-of-the-art in program repair' broadly, but results are only on Java benchmarks (Defects4j, QuixBugs-Java, HumanEval-Java) and one Python dataset (QuixBugs-Python). The approach is presented as general but tested in a narrow setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section discusses dataset selection and data leakage but does not discuss alternative explanations for why contrastive pairs help. For instance, the improvement could be due to simply providing more information in the prompt rather than the contrastive nature specifically."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 states 'gpt-3.5-turbo-0301' — a specific versioned model identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows a prompt template with placeholders (e.g., '{Error Message in Traceback}') but does not provide the actual fill values or complete prompts used. The template alone is insufficient to reconstruct exact prompts sent."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 reports temperature=1, n=3, m=40, similarity threshold θ=0.5, 1000 mutated test cases, 30-second timeout per test case evaluation."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The conversational repair loop (Algorithm 1), restart/continue strategies, pair selection, prompt construction, and patch augmentation are all described in detail in Sections 3.2-3.3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 describes the instrumentation process to capture parameter values, the mutation process, and how test cases are collected. Section 4.1.2 describes dataset selection and fault localization setup."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 'Threats to Validity' provides substantive discussion of multiple threats including dataset selection, data leakage, manual verification, and efficiency measurement."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats include: re-implementation of CHATREPAIR may differ from original; hyperparameter settings differ from original CHATREPAIR paper (200 vs 40 restarts); manual patch verification requires consensus among three reviewers; some baseline results taken from other papers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to specific languages, bug types, or project sizes. The limitation about logical bugs lacking test oracles is mentioned but framed as future work rather than a scope boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Generated patches are said to be open-sourced, but no link is provided in the paper. Raw experimental logs, API call records, and generated test cases are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2-4.1.3 describes how bugs are collected from Defects4j, QuixBugs, and HumanEval-Java, including version splits and localization setup."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from test collection through instrumentation, mutation, pair selection, prompting, and patch validation is documented in Sections 3 and 4.1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments or grant information is mentioned in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Singapore Management University, Nanyang Technological University, Monash University, and Tianjin University."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges data leakage risk and uses HumanEval-Java as a post-training dataset, but does not state the exact training cutoff date for gpt-3.5-turbo-0301."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 (RQ2) and Section 5.2 explicitly discuss data leakage concerns with Defects4j and QuixBugs potentially being in ChatGPT's training data, and evaluate on HumanEval-Java to mitigate this."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges Defects4j and QuixBugs may be contaminated and uses HumanEval-Java (published January 2023, after GPT-3.5 training) as an uncontaminated benchmark (Section 4.3)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study; only manual patch verification by researchers."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports average number of API queries per bug (#Query) across all experiments (Tables 4, 5, 6, 7, 8), and notes mutation time is 15-25 minutes per bug."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total API cost in dollars, total GPU hours, or aggregate compute budget is stated. Only per-bug query counts are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "ContrastRepair correctly repairs 143 out of 337 bugs on Defects4j 1.2 and 2.0 combined, surpassing the best baseline CHATREPAIR which fixes 124.",
       "evidence": "Table 1 shows 103 fixes on D4J1.2 and 40 on D4J2.0 (total 143). CHATREPAIR gets 90+34=124.",
       "supported": "strong"
     },
     {
       "claim": "ContrastRepair reduces API calls by ~20.91% compared to CHATREPAIR.",
       "evidence": "Table 4 shows #Query reductions of 28.83% (SL), 30.20% (SH), and 5.97% (SF) on D4J1.2. Table 5 shows 29.34% reduction on HumanEval-Java.",
       "supported": "strong"
     },
     {
       "claim": "ContrastRepair fixes all 40 bugs in QuixBugs-Java and QuixBugs-Python.",
       "evidence": "Table 2 shows 40/40 for both Python and Java.",
       "supported": "strong"
     },
     {
       "claim": "Contrastive test pairs provide more useful information than failing tests alone.",
       "evidence": "Table 6 ablation: 2 pairs → 37 correct fixes vs 2 failed cases → 33 fixes. Table 8: w/o Pair → 30.67 vs ContrastRepair → 37.00 correct fixes.",
       "supported": "strong"
     },
     {
       "claim": "Similarity-based pair selection outperforms random selection and semantic-based approaches.",
       "evidence": "Table 8: w/o Similarity → 33.33 vs ContrastRepair → 37.00. Table 10: Damerau-Levenshtein outperforms BM25, CodeBERT, UniXcoder.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ContrastRepair enhances conversation-based automated program repair by providing LLMs with contrastive test case pairs (a failing test paired with a similar passing test). Evaluated on Defects4j, QuixBugs, and HumanEval-Java using gpt-3.5-turbo-0301, it fixes 143/337 Defects4j bugs versus 124 for CHATREPAIR, while requiring ~20% fewer API calls. Ablation studies confirm that both the contrastive pair mechanism and similarity-based selection contribute meaningfully to performance.",
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper claims 'significant' improvements throughout but never performs any statistical test. All comparisons are based on raw count differences without assessing whether differences could arise by chance."
     },
     {
       "flag": "Variance only partially reported",
       "detail": "Ablation experiments (RQ4) are repeated 3 times and averages reported, but no standard deviations are given. Main experiments (RQ1, RQ2) appear to be single runs despite the stochastic nature of ChatGPT sampling at temperature=1."
     },
     {
       "flag": "Re-implemented baseline",
       "detail": "CHATREPAIR was re-implemented since it was not open-sourced. The authors acknowledge a gap between their reproduction and the original paper's results (different hyperparameter settings: 40 vs 200 restarts), which may disadvantage the baseline."
     },
     {
       "flag": "Perfect fault localization assumption",
       "detail": "All experiments use perfect (ground truth) fault localization, which is unrealistic. The paper acknowledges this is more practical at function level but still provides oracle-level information."
     }
   ],
   "cited_papers": [
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "Primary baseline; the first fully automated conversation-driven APR approach using ChatGPT."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "doi": "10.1145/3540250.3549101",
       "relevance": "AlphaRepair, the first cloze-style APR approach using pre-trained code models without fine-tuning."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Studies the impact of LLMs on APR and introduces HumanEval-Java benchmark used in this paper."
     },
     {
       "title": "Conversational automated program repair",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2301.13246",
       "relevance": "Early work on conversational APR with LLMs."
     },
     {
       "title": "An analysis of the automatic bug fixing performance of chatgpt",
       "authors": ["Dominik Sobania", "Martin Briesch", "Carol Hanna", "Justyna Petke"],
       "year": 2023,
       "arxiv_id": "2301.08653",
       "relevance": "Evaluates ChatGPT's bug-fixing capabilities on QuixBugs."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark for LLM code generation evaluation."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "relevance": "Studies LLM-based automated program repair at ICSE 2023."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Rigorous evaluation of LLM code generation correctness."
     },
     {
       "title": "A Survey of Learning-based Automated Program Repair",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Yuxiang Ma", "Weisong Sun", "Zhenyu Chen"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Comprehensive survey of learning-based APR methods."
     }
   ]
 }