ai-research-survey

scan.json (25149B)

---

 {
   "paper": {
     "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
     "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
     "year": 2023,
     "venue": "arXiv / ISSTA (submitted)",
     "arxiv_id": "2304.00385",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided in the paper. The paper describes the implementation in Python accessing the ChatGPT API but does not release the source code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: Defects4j (v1.2 and v2.0) and QuixBugs, both well-known public datasets with established download procedures."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies the evaluation environment: '8-core workstation with Intel i7 10700KF Comet Lake CPU @3.80GHz and 64GB RAM, running Ubuntu 20.04.3 LTS and OpenJDK Java 64-Bit Server version 1.8.0_312.' The model used is also specified as gpt-3.5-turbo-0301."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. While the approach is described algorithmically (Algorithm 1), there are no concrete commands or reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (number of bugs fixed). No confidence intervals or error bars are provided despite the stochastic nature of LLM sampling."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ChatRepair outperforms baselines (e.g., '15 and 17 more than prior best baseline') but no statistical significance tests are performed. Comparisons are based solely on raw counts of bugs fixed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements in absolute terms with context: '114 and 48 correct bug fixes (15 and 17 more than prior best baseline) on Defects4j 1.2 and 2.0 respectively.' The ablation study also reports specific numeric improvements (e.g., plausible patch generation adds 4, 7, 2 correct fixes in the three settings)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes. The benchmarks used (391 bugs in D4J 1.2, 82 single-line in D4J 2.0, 40 in QuixBugs) are inherited from prior work without discussion of whether they are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported from what appears to be a single run. No standard deviation, variance, or multi-run results are reported despite the stochastic nature of LLM sampling with temperature=1."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against 21 prior APR tools including 8 learning-based/LLM-based baselines (SelfAPR, AlphaRepair, RewardRepair, Recoder, CURE, CoCoNuT, DLFix, SequenceR) and 12 traditional APR tools plus a BaseChatGPT ablation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary tools: AlphaRepair (2022), SelfAPR (2022), RewardRepair (2022), CodexRepair (2023), all published within 1-2 years of this work. These represent the state of the art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.3 (RQ3) provides a comprehensive ablation study examining: (1) initial prompt variations (Table 4), (2) feedback response variations (Table 5), and (3) maximum conversation length (Figure 7). Each component's contribution is measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: plausible patches, correct patches, number of tries (queries to ChatGPT), and dollar cost. Section 4.3 describes all four metrics."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'We follow common practice in APR and manually determine the semantic equivalency to compute correct patches.' This manual inspection of patches by the authors constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Defects4j 2.0 (438 new bugs from 9 additional projects) serves as a separate evaluation set from D4J 1.2. QuixBugs provides a third independent benchmark. The ablation study uses the 80 single-line D4J 1.2 bugs separately."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides per-project breakdowns (Chart, Closure, Lang, Math, Mockito, Time) for both D4J 1.2 and 2.0. Table 3 breaks down by repair scenario (single-line, single-hunk, single-function). The Venn diagram (Figure 3) shows overlap with other tools."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper primarily shows success cases (Figures 4, 5, 6). While some discussion of when ChatRepair fails is implicit (e.g., only 114/391 bugs fixed on D4J 1.2), no explicit failure analysis or qualitative examples of failures are provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that BaseChatGPT 'performs even slightly worse than CodexRepair on the real-world benchmark of Defects4j 1.2' (Section 5.2), and the ablation shows that some configurations perform worse (e.g., BasePrompt achieves fewer fixes than the full approach)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '114 and 48 correct fixes on Defects4j 1.2 and 2.0 respectively' and '$0.42 each' — these are directly supported by Table 1 and the cost analysis in Section 5.1. The claim of 'new state-of-the-art' is supported by the comparison tables."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about components improving performance (e.g., test failure information helps, conversation helps). The ablation study in Section 5.3 systematically varies individual components while controlling others (Tables 4, 5, Figure 7), providing adequate single-variable manipulation evidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract present broad claims ('conversation-driven APR approach') but the evaluation is limited to Java bugs on Defects4j and QuixBugs using a single LLM (gpt-3.5-turbo-0301). While the abstract says 'our approach is general,' the generalization to other languages, models, or bug types is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Threats to Validity) discusses the alternative explanation that performance may come from data leakage (training data memorization). The paper addresses this by showing 36% of patches match developer fixes, and ChatRepair still outperforms BaseChatGPT using the same model (ruling out pure memorization)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'We use the gpt-3.5-turbo-0301 model of the ChatGPT family which is the current latest model available to us.' This is a specific model version with a date identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the actual prompt templates with system message ('You are an Automated Program Repair Tool'), the initial prompt structure with infill notation, feedback messages, and the plausible patch generation instruction ('Please generate an alternative fix line'). The prompts are shown with actual fill values in the examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: sampling temperature of 1, maximum repair attempts of 200 (single-line/hunk) and 100 (single-function), 1 few-shot example, maximum conversation length of 3, and 5-hour end-to-end timeout."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The conversational scaffolding is described in detail in Section 3 with Algorithm 1: the feedback loop (compile & test → construct feedback → re-query), plausible patch generation (Section 3.3), conversation length management, and retry logic are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents how bugs are categorized from Defects4j (e.g., 'we categorize Defects4j 1.2 into single-function (255 bugs), single-hunk (154 bugs) and single-line (80 bugs)' with subset relationships explained). Section 3.1 describes how prompts are constructed from buggy code."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' discusses both internal threats (manual validation correctness, data leakage) and external threats (generalizability to other datasets)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 provides specific threats: (1) manual validation bias with the mitigation of careful examination, (2) data leakage quantified at 36% of patches matching developer fixes, and (3) generalizability limited to Defects4j and QuixBugs with mitigation of using multiple datasets."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the threats section mentions dataset limitations, the paper does not explicitly state what the results do NOT show. There is no statement bounding the scope to Java, to the specific ChatGPT version used, or to the specific types of bugs tested. The title and claims suggest broad applicability without explicit boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw patches generated, conversation logs, and per-bug cost breakdowns are not made available. Only aggregate results are reported in the tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection process is well described: Defects4j and QuixBugs are established benchmarks with known provenance. Section 4.2 describes the bug selection process and categorization. The evaluation procedure (compile and test each generated patch) is described in detail."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The study uses standard public benchmarks (Defects4j, QuixBugs)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: buggy code → prompt construction (Section 3.1) → ChatGPT query → patch extraction → compile and test validation → feedback loop (Algorithm 1). The categorization of bugs into repair scenarios is explained in Section 4.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. The authors are from UIUC, an academic institution, but no grants or sponsorships are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: both authors are from the University of Illinois Urbana-Champaign. The paper evaluates OpenAI's ChatGPT but the authors have no disclosed affiliation with OpenAI."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed at all. Without knowing who funded the research, independence cannot be confirmed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses gpt-3.5-turbo-0301 to evaluate on Defects4j and QuixBugs benchmarks, but does not state the model's training data cutoff date. Section 6 acknowledges 'ChatGPT is a proprietary model and can only be accessed through API, we do not have access to the exact training data used.'"
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly discusses this: 'Another threat to validity comes from the data leakage of reference developer patches being part of the original training data of ChatGPT.' The paper quantifies that 36% of correct patches match developer fixes and shows ChatRepair still outperforms BaseChatGPT, arguing the improvement is not from memorization."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 directly addresses this by analyzing what percentage of correct patches are identical to developer fixes (36%), showing ChatRepair fixes 12 unique bugs no prior tool can fix even after removing memorized patches, and comparing against BaseChatGPT to isolate the contribution of the method vs. memorization."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. It is a benchmark evaluation of an automated tool."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost is a central metric. The paper reports $0.42 per bug on average, and the ablation study (Tables 4, 5) reports average dollar cost per configuration. Section 4.3 explains the cost model: '$0.002 per every 1000 tokens processed or generated.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states the hardware (Intel i7, 64GB RAM), the 5-hour timeout per bug, and the maximum number of API queries (200 for single-line/hunk, 100 for single-function). The total cost per bug is reported ($0.42)."
       }
     }
   },
   "claims": [
     {
       "claim": "ChatRepair achieves 114 and 48 correct bug fixes on Defects4j 1.2 and 2.0 respectively, surpassing all prior APR tools by 15 and 17 fixes.",
       "evidence": "Table 1 shows ChatRepair at 114 on D4J 1.2 (vs. CodexRepair 99, next best) and 48 on D4J 2.0 (vs. AlphaRepair 36, next best). Section 5.1 discusses these results.",
       "supported": "strong"
     },
     {
       "claim": "ChatRepair can fix 162 out of 337 bugs for $0.42 each using ChatGPT.",
       "evidence": "Section 5.1 reports the combined count and cost. The cost model is described in Section 4.3 based on the ChatGPT API pricing at the time.",
       "supported": "moderate"
     },
     {
       "claim": "ChatRepair fixes 18 unique bugs on Defects4j 1.2 that no prior approach can fix.",
       "evidence": "Figure 3 (Venn diagram) shows 18 bugs uniquely fixed by ChatRepair. Examples are discussed in Figures 4 and 5.",
       "supported": "strong"
     },
     {
       "claim": "Test failure information and conversational feedback each contribute to ChatRepair's improvement over baseline ChatGPT.",
       "evidence": "Section 5.3 (RQ3) ablation study: Table 4 shows BasePrompt (55 plausible) vs. full prompt with test info (64 plausible). Table 5 shows BaseFeedback (58) vs. Dynamic (64). Figure 7 shows conversation length=1 (no conversation) performs worst.",
       "supported": "strong"
     },
     {
       "claim": "Plausible patch generation adds on average 9.4, 16.6, and 5.5 plausible patches in single-line, single-hunk, and single-function scenarios.",
       "evidence": "Section 5.2 reports these numbers and states the plausible patch generation step improves correct fixes by 4, 7, 2 in the respective scenarios on D4J 1.2.",
       "supported": "moderate"
     },
     {
       "claim": "ChatRepair correctly fixes all 40 bugs in QuixBugs-Python and all 40 in QuixBugs-Java.",
       "evidence": "Table 2 shows 40/40 for both Python and Java QuixBugs datasets.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "ChatRepair introduces a conversation-driven approach to automated program repair that feeds test failure information and prior patch attempts back to ChatGPT (gpt-3.5-turbo-0301) in an iterative loop. On Defects4j 1.2 and 2.0, it achieves 114 and 48 correct fixes respectively, surpassing the prior state-of-the-art by 15 and 17 fixes. The ablation study shows that both test failure information in the initial prompt and conversational feedback with dynamic error messages contribute to improved repair performance. The approach achieves these results at a cost of $0.42 per bug on average.",
   "red_flags": [
     {
       "flag": "No variance or multi-run reporting",
       "detail": "LLM sampling with temperature=1 is inherently stochastic, yet the paper appears to report single-run results with no standard deviation, confidence intervals, or multiple-run analysis. Bug counts could vary meaningfully across runs."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of superiority over baselines are based solely on raw bug-count comparisons (e.g., 114 vs. 99) without any significance testing, despite the stochastic nature of the approach."
     },
     {
       "flag": "Missing failure analysis",
       "detail": "The paper fixes 114 out of 391 bugs on D4J 1.2 (29%) but provides no analysis of why the remaining 277 bugs were not fixed. Only success cases are qualitatively analyzed (Figures 4, 5, 6)."
     },
     {
       "flag": "Potential benchmark contamination",
       "detail": "Defects4j bugs and fixes have been publicly available since 2014. The paper acknowledges that 36% of correct patches match developer fixes exactly, raising contamination concerns. While addressed in threats to validity, this percentage is notable."
     },
     {
       "flag": "Baseline fairness concerns",
       "detail": "Most baseline results are taken from prior publications that used different models and potentially different configurations. Only BaseChatGPT and CodexRepair provide a controlled comparison using the same or similar LLMs under comparable conditions."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex paper; one of the baselines (CodexRepair) in this study uses Codex for code generation."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Prior study by same authors extensively evaluating LLM-based APR across multiple models and architectures; provides the CodexRepair baseline."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Introduces AlphaRepair (cloze-style APR using CodeBERT), a key baseline and state-of-the-art LLM-based APR tool evaluated in this paper."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "GPT-3 paper establishing the few-shot prompting paradigm used as foundation for LLM-based program repair approaches."
     },
     {
       "title": "SelfAPR: Self-supervised Program Repair with Test Execution Diagnostics",
       "authors": ["He Ye", "Matias Martinez", "Xiapu Luo", "Tao Zhang", "Martin Monperrus"],
       "year": 2022,
       "relevance": "NMT-based APR baseline that also uses test execution information, enabling comparison with ChatRepair's prompting-based approach."
     },
     {
       "title": "Neural Program Repair with Execution-based Backpropagation",
       "authors": ["He Ye", "Matias Martinez", "Martin Monperrus"],
       "year": 2022,
       "relevance": "RewardRepair baseline that uses test execution rewards for training NMT repair models."
     },
     {
       "title": "A Syntax-Guided Edit Decoder for Neural Program Repair",
       "authors": ["Qihao Zhu", "Zeyu Sun", "Yuan-an Xiao"],
       "year": 2021,
       "relevance": "Recoder: syntax-guided neural program repair baseline compared in the evaluation."
     },
     {
       "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "NMT-based APR baseline using code-aware translation for automated program repair."
     },
     {
       "title": "Can OpenAI's Codex Fix Bugs?: An evaluation on QuixBugs",
       "authors": ["Julian Aron Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "relevance": "Early evaluation of LLM (Codex) for program repair on QuixBugs benchmark."
     },
     {
       "title": "Is the cure worse than the disease? overfitting in automated program repair",
       "authors": ["Edward K Smith", "Earl T Barr", "Claire Le Goues", "Yuriy Brun"],
       "year": 2015,
       "relevance": "Foundational work on overfitting in APR — the plausible-but-incorrect patch problem that ChatRepair's plausible patch generation step aims to mitigate."
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "authors": ["Daniel Fried", "Armen Aghajanyan", "Jessy Lin"],
       "year": 2022,
       "arxiv_id": "2204.05999",
       "relevance": "Code infilling model used as a baseline LLM in prior APR studies compared with ChatRepair."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeffrey Wu", "Xu Jiang"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF paper foundational to ChatGPT's instruction-following capability leveraged by ChatRepair."
     }
   ]
 }