ai-research-survey

scan.json (25476B)

---

 {
   "paper": {
     "title": "Aligning the Objective of LLM-based Program Repair",
     "authors": [
       "Junjielong Xu",
       "Ying Fu",
       "Shin Hwei Tan",
       "Pinjia He"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2404.08877",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our source code and experimental results are publicly available at https://github.com/CUHK-Shenzhen-SE/D4C' in the Data Availability footnote at the end of Section VII."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks (Defects4J and DebugBench), both of which are established public benchmarks. The experimental results are also released at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Python 3.9', 'Ubuntu 20.04.5 LTS', and '8xA100 NVIDIA GPU server' in Section IV-A, but does not provide a requirements.txt, Dockerfile, or detailed dependency specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references the GitHub repository but does not include step-by-step reproduction instructions in the paper itself. The reader is directed to the repo without a description of how to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports only point estimates (number of correct patches) in Tables II-V. No confidence intervals or error bars are reported for any main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims D4C 'outperforms' baselines and 'surpasses the SOTA APR methods by 10%' but provides no statistical significance tests. Comparisons are made purely by comparing raw counts of correct patches."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline context for its improvements: '180 bugs correctly in Defects4J... surpasses the SOTA APR methods with perfect fault localization by 10%' (abstract), and Table II shows all baselines' counts alongside D4C's, giving the reader sufficient context to assess the magnitude (180 vs 162 for ChatRepair, a ~11% improvement)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates on 437 single-function bugs from Defects4J and 590 from DebugBench, but there is no justification for why these sample sizes are adequate for the claims made, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main results (Tables II-V) report single-run numbers with no variance across runs. The only standard deviation reported is for the position of the correct patch among 10 samples (std=1.74 in Sec. V-C) and plausible patches per bug (std=3.28 in Sec. V-D), but these are descriptive statistics about patch distributions, not variance across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table II compares D4C against five state-of-the-art LLM-based APR methods: AlphaRepair, Repilot, RAP-Gen, FitRepair, and ChatRepair."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines include ChatRepair (2023, GPT-4 based), RAP-Gen (2023), FitRepair (2023), and Repilot (2023), all of which are recent and represent the state of the art for LLM-based APR at the time of submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section IV-D, Table V) provides a thorough ablation study removing individual components: w/o Document, w/o Test, w/o Message, -Mask, and -Pure, quantifying each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses multiple metrics: number of correct patches (manually verified), number of plausible patches (test-passing), number of verified patches (LeetCode unseen tests), perplexity for objective alignment validation, and patch sampling efficiency."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes manual validation of patch correctness: 'Two authors independently confirm the patch correctness. Any patches with disagreement were presented to the third author for review' (Section V-A, Internal Threats). This is human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "DebugBench uses LeetCode unseen test suites for verification that are separate from the test examples provided to the model. For Defects4J, the standard test suite is used for validation, and the paper explicitly addresses data leakage by using DebugBench for insight validation (RQ2)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by Defects4J v1.2 vs v2.0 (Table II), by programming language (C++/Java/Python in Table IV and V for DebugBench), and by perfect vs statistical FL settings (Table III)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-B provides qualitative analysis of plausible-but-incorrect patches, including a specific example (Fig. 7, Jsoup-19) where the LLM generated a patch that passed tests but was semantically incorrect due to overfitting to the failed test case."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that D4C performs worse than ChatRepair on Defects4J v1.2 (84 vs 114 in Table II), and that setting temperature to 0 results in fewer patches. The '-Pure' and '-Mask' ablations show degraded performance. Also, the paper discusses that 30 out of 210 plausible patches were incorrect."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '180 bugs correctly in Defects4J' (supported by Table II sum column), 'surpasses the SOTA APR methods with perfect fault localization by 10%' (180 vs 162 ChatRepair), and 'reduces the patch sampling number by 90%' (10 vs 100). All are supported by the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about objective alignment improving performance. It supports this through controlled ablation experiments (RQ2, Table IV) comparing different input/output formats while controlling other variables, and validates via perplexity measurements on a white-box model. The ablation design constitutes adequate controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'our results may not generalize beyond the studied settings and other programming languages beyond the supported ones in Defects4J and DebugBench' (Sec. V-A, External Threats). The title 'LLM-based Program Repair' is appropriately bounded to the evaluated domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses the alternative explanation that improvements could be due to data leakage rather than objective alignment, using DebugBench (leakage-free) to validate insights. Section V-A discusses that model effectiveness may vary across settings, and data leakage is discussed at length in Sections IV-A and IV-B."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A specifies 'gpt-4-0613' and 'mixtral-8x7b-instruct-v0.1' as the exact model versions used, including that they used 'fixed remote API checkpoints or fixed local model versions.'"
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper shows the prompt structure in Fig. 4 and Fig. 5 with templates using placeholders like {BUGGY_DOCUMENT}, {FAILED_TEST}, {TEST_INFO}, {BUGGY_CODE}, and a system instruction ('As an debugger, you should refine the buggy program for bug report'). However, the complete actual prompt text with the fixed example pair is not provided in the paper. The fixed example is described but not fully shown."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A reports temperature=1.0, sampling number=10, and timeout threshold of 1 minute per patch. Section IV-D additionally explores temperature=0.0 and different sampling numbers (1, 3, 10)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "D4C is a single-pass prompting framework without agentic scaffolding. It does not use multi-round dialogue, tool use, retry logic, or feedback mechanisms. It is a one-shot prompting approach."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-C describes the artifact extraction process (extracting documents, failed test cases, and error messages). Section IV-A explains how single-function bugs were separated from Defects4J (v1.2: 203, v2.0: 234) and how DebugBench logic bugs were selected (590 out of 4,253 total bugs). The filtering criteria are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V-A 'Threats to Validity' provides a dedicated subsection discussing external and internal threats with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: data leakage of Defects4J in LLM pre-training, incomplete test coverage of Defects4J not guaranteeing patch correctness, potential bias in manual patch validation, and that FLUCCS only supports Defects4J v1.2. These are concrete, study-specific threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states: 'our results may not generalize beyond the studied settings and other programming languages beyond the supported ones in Defects4J and DebugBench' (Sec. V-A). It also clarifies that 'This paper is not aimed at proposing D4C as a new APR technique, but rather to introduce a new mindset or paradigm' (Sec. I)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we decide to release our experimental results for public verification' (Sec. V-A) and provides a GitHub repository. Both Defects4J and DebugBench are publicly available benchmarks."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-A describes the data collection in detail: 437 single-function bugs from Defects4J (v1.2: 203, v2.0: 234), 590 logic bugs from DebugBench (200 C++, 194 Java, 196 Python), and explains the selection criteria (logic bugs, single-function bugs)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks (Defects4J and DebugBench)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: identify buggy function (via method-level FL), extract artifacts (documents, failed tests, error messages per Sec. III-C), construct prompt (Sec. III-D), generate patches (Sec. III-E), validate patches via test suite then manual verification (Sec. III-F). Each stage is described with counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section VII (Acknowledgment) discloses funding: 'Guangdong Basic and Applied Basic Research Foundation (No. 2024A1515010145) and the Shenzhen Science and Technology Program (No. ZDSYS20230626091302006).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: The Chinese University of Hong Kong, Shenzhen; Chongqing University; and Concordia University. None of the authors are affiliated with OpenAI or Mistral, the companies whose products are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders are Chinese government research foundations (Guangdong Basic and Applied Basic Research Foundation, Shenzhen Science and Technology Program), which have no financial interest in the outcome of this APR research."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff date for either GPT-4 (gpt-4-0613) or Mixtral-MoE. While data leakage is discussed extensively, the actual training cutoff dates are never specified."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses train/test overlap: it acknowledges Defects4J data may be in LLM training corpora, argues that bug-fix pairs are sparse in training data (Sec. IV-A, IV-B), and uses DebugBench specifically because it was 'designed to counter data leakage (by implanting bugs into source data with GPT-4)' (Sec. IV-A)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses this concern: Table I labels Defects4J as 'Yes' for data leakage risk and DebugBench as 'No'. The authors use DebugBench for insight validation (RQ2, RQ3) specifically to mitigate contamination: 'we selected DebugBench, a latest, leakage-free benchmark, for evaluation' (Sec. V-A)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study. The manual validation of patches is done by the paper authors, not as a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C reports detailed cost information: average input prompt length (1,386.84 tokens), average output length (314.39 tokens), average cost per patch ($0.023), total cost for 10 patches per bug ($0.23 for the entire dataset), and reduced cost if stopping at first correct patch ($0.18 per bug)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While API cost per patch is reported for GPT-4, no total GPU hours or compute budget is stated for the Mixtral-MoE experiments run on the 8xA100 server. The overall compute budget for the full experimental campaign is not quantified."
       }
     }
   },
   "claims": [
     {
       "claim": "D4C repairs 180 bugs correctly in Defects4J, surpassing SOTA APR methods with perfect fault localization by 10%.",
       "evidence": "Table II shows D4C fixes 180 bugs total (84 on v1.2 + 96 on v2.0) vs ChatRepair's 162 (114 + 48), using only 10 samples per bug vs 100-5000 for baselines.",
       "supported": "moderate"
     },
     {
       "claim": "Generating a complete refined function aligns better with the decoder-only LLM's training objective than generating fixed hunks.",
       "evidence": "Table IV shows output perplexity for complete function (Func) is consistently lower than for hunks (Hunk): Report-Func output perplexity is 1.39 vs Report-Hunk at 8.50 on Mixtral-MoE. Report-Func also generates more verified patches across all languages and models.",
       "supported": "strong"
     },
     {
       "claim": "Providing artifacts (documents, failed tests, error messages) enables LLMs to locate and repair buggy hunks without statement-level fault localization.",
       "evidence": "Table IV shows Report-Func generates more verified patches than Mask-Func across all languages (e.g., 118 vs 99 C++, 125 vs 91 Java on Mixtral-MoE). Table V ablation shows removing any artifact component reduces performance.",
       "supported": "strong"
     },
     {
       "claim": "D4C is less affected by inaccuracies in fault localization than infilling-style methods.",
       "evidence": "Table III shows D4C drops only 4.8% from perfect to statistical FL (84 to 80), while AlphaRepair drops 30.8% and RAP-Gen drops 33.3%.",
       "supported": "moderate"
     },
     {
       "claim": "D4C requires only 10 patch samples per bug, which is 90% fewer than the most efficient baseline.",
       "evidence": "Table II shows D4C uses 10 samples while RAP-Gen uses 100, ChatRepair uses 100-200, and AlphaRepair/Repilot/FitRepair use 5000.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "D4C demonstrates that aligning the output format of decoder-only LLMs to their training objective (complete function generation vs. hunk infilling) significantly improves automated program repair performance. On Defects4J, D4C fixes 180 bugs using only 10 samples per bug, outperforming the best baseline (ChatRepair, 162 bugs with 100-200 samples). Perplexity analysis on Mixtral-MoE validates that function completion has substantially lower perplexity than hunk generation, providing direct evidence for the objective alignment hypothesis. The approach also shows greater robustness to imperfect fault localization compared to infilling methods.",
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons are based on raw counts of correct patches without any significance tests. The difference between D4C (180) and ChatRepair (162) is presented as 'surpassing by 10%' but the statistical reliability of this difference is never tested."
     },
     {
       "flag": "Single-run results with no variance reporting",
       "detail": "Despite using temperature=1.0 (stochastic decoding), the main results appear to be from a single run. No repeated trials with different random seeds are reported, making it impossible to assess result stability."
     },
     {
       "flag": "Uneven comparison conditions",
       "detail": "D4C uses GPT-4 while most baselines use weaker models (CodeBERT, CodeT5, InCoder). Only ChatRepair also uses GPT-4. The 'surpasses SOTA by 10%' claim conflates model choice with method design, though the paper partially addresses this via controlled ablations in RQ2."
     },
     {
       "flag": "Baseline results from original papers without reproduction",
       "detail": "The paper acknowledges 'we reuse their Defects4J results reported in the original paper' (Sec. IV-B) because some baselines are not open-sourced. Different experimental conditions (hardware, API versions, prompting strategies) across papers may affect comparability."
     }
   ],
   "cited_papers": [
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "Key baseline (AlphaRepair) that pioneered infilling-style APR using LLMs, directly compared to D4C."
     },
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "ChatRepair is the strongest baseline using GPT-4 with multi-round dialogue for APR, directly compared in Table II."
     },
     {
       "title": "Rap-gen: Retrieval-augmented patch generation with codet5 for automatic program repair",
       "authors": ["W. Wang", "Y. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2023,
       "arxiv_id": "2309.06057",
       "relevance": "RAP-Gen is a retrieval-augmented LLM-based APR method compared as a baseline."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Y. Wei", "C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2309.00608",
       "relevance": "Repilot is an LLM-based APR baseline that fuses LLMs with completion engines, directly evaluated."
     },
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["X. Chen", "M. Lin", "N. Schärli", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Self-debugging approach that inspired D4C's concept of allowing LLMs to refine programs using execution feedback."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Demonstrates that data leakage is less severe for APR compared to other code tasks, a key argument for D4C's Defects4J evaluation."
     },
     {
       "title": "DebugBench: Evaluating debugging capability of large language models",
       "authors": ["R. Tian", "Y. Ye", "Y. Qin", "X. Cong", "Y. Lin", "Z. Liu", "M. Sun"],
       "year": 2024,
       "arxiv_id": "2401.04621",
       "relevance": "Key benchmark used for insight validation, designed to be leakage-free by implanting bugs with GPT-4."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Z. Fan", "X. Gao", "M. Mirchev", "A. Roychoudhury", "S. H. Tan"],
       "year": 2023,
       "relevance": "LLM-based APR approach using flexible fault localization, which supports D4C's insight about not restricting repair to specific hunks."
     },
     {
       "title": "Revisiting the plastic surgery hypothesis via large language models",
       "authors": ["C. S. Xia", "Y. Ding", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2303.10494",
       "relevance": "FitRepair baseline that revisits APR with LLMs, directly compared to D4C."
     },
     {
       "title": "A unified debugging approach via LLM-based multi-agent synergy",
       "authors": ["C. Lee", "C. S. Xia", "L. Yang", "J.-t. Huang", "Z. Zhu", "L. Zhang", "M. R. Lyu"],
       "year": 2024,
       "arxiv_id": "2404.17153",
       "relevance": "Multi-agent LLM-based debugging approach that represents the expanding scope of LLM use in program repair."
     },
     {
       "title": "A survey of learning-based automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Ma", "W. Sun", "Z. Chen"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Comprehensive survey of learning-based APR methods providing context for the field D4C operates in."
     }
   ]
 }