ai-research-survey

scan.json (19844B)

---

 {
   "paper": {
     "title": "Copiloting the Copilots: Fusing Large Language Models with Completion Engines for Automated Program Repair",
     "authors": ["Yuxiang Wei", "Chunqiu Steven Xia", "Lingming Zhang"],
     "year": 2023,
     "venue": "ESEC/FSE 2023",
     "arxiv_id": "2309.00608",
     "doi": "10.1145/3611643.3616271"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub link provided: https://github.com/ise-uiuc/Repilot. Also an immutable Zenodo artifact is mentioned in the Data Availability section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available Defects4J 1.2 and 2.0 benchmarks. Correct patches are released for public evaluation (reference [62], Zenodo)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 specifies hardware (32-Core Ryzen Threadripper PRO 3975WX, 256 GB RAM, NVIDIA RTX A6000), OS (Ubuntu 20.04.4 LTS), Java version (OpenJDK 1.8.0_181), and models from Hugging Face. Python implementation with 5K lines of code and modified Eclipse JDT LS in Java with 1.5K lines."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code is released, the paper itself does not include step-by-step reproduction instructions. The Zenodo artifact and GitHub repo may contain them, but the paper does not describe reproduction steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (number of correct fixes, compilation rates). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Repilot 'outperforms' baselines based on comparing raw numbers (e.g., 66 vs 52 fixes) without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: '27% and 47% more bugs' fixed, percentage point improvements in compilation rate (e.g., 43.2% to 63.4%), and absolute counts for both Repilot and baselines."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 138 and 135 single-hunk bugs are sufficient, or why 5000/500 samples per bug were chosen beyond 'fair comparison' and 'high cost'."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 (Threats to Validity) acknowledges 'we only run each of our experiments once, which could introduce extra statistical biases.' No variance or standard deviation across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "19 APR baselines are compared: 12 traditional, 6 NMT-based, and 1 LLM-based (AlphaRepair). Listed in Section 5.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent tools like AlphaRepair (2023), RewardRepair (2022), Recoder (2022), representing state-of-the-art at the time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 6.3) presents a detailed ablation study with four variants: Repilot∅ (base LLM only), Repilotp (with pruning), Repilotmp (with memorization), and full Repilot (with active completion)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics used: number of correct fixes, number of plausible fixes, compilation rate (%), plausible rate (%), and generation time."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Correct patches are determined by manual examination: 'we determine semantic equivalency by manually examining each plausible patch' (Section 5.4)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a learning/training evaluation — Repilot uses pre-trained models on a fixed benchmark. No train/test split concept applies to the evaluation itself."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Defects4J 1.2 vs 2.0), by model (CodeT5 vs InCoder), and by bug type (single-hunk vs single-line). Venn diagrams show unique fixes per tool."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Limitations) discusses when Repilot fails: dynamically typed languages, higher compilation rate not proportionally increasing correct fixes, and lack of evidence for rare token claims."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 acknowledges that 'a significantly higher compilation rate does not necessarily translate to a proportionally large increase in plausible and correct fixes.' The ablation also shows overhead costs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims '27% and 47% more bugs' are supported by Table 1 (66 vs 52 for D4J 1.2 = 27%, 50 vs 34 for D4J 2.0 = 47%). Compilation rate and validity claims are supported by Tables 2-3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by the ablation study (RQ3) which systematically adds components. The ablation design (controlled single-variable manipulation) is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly bounds scope: 'the scope of our evaluation considering two LLMs (CodeT5 and InCoder) and one programming language (Java) is still narrow.' Also acknowledges difficulty with dynamically typed languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses training data overlap as an alternative explanation (7/66 and 6/50 bugs overlap with CodeT5 training data) and provides adjusted numbers excluding overlapping bugs."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific models named: CodeT5-large (770M params) and InCoder-6.7B, obtained from Hugging Face. These are specific model identifiers with sizes."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "Repilot does not use prompting — it uses cloze-style infilling where buggy code is replaced with a mask token and the LLM fills it in. This is a model inference approach, not a prompting approach."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1: top-p nucleus sampling with p=1.0, temperature=1.0, max_tokens=50, 5000 samples per bug (RQ1/RQ2) or 500 samples (RQ3/RQ4), 5-hour timeout per bug."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper describes the scaffolding in detail: the interaction loop between LLM and Completion Engine (Algorithms 1-3), pruning, memorization (Trie), and active completion are all formally specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 documents data selection: starting from Defects4J, filtering to single-hunk bugs, removing 4 deprecated bugs, removing bugs incompatible with Completion Engine, resulting in 138 and 135 bugs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Dedicated Section 7 (Limitations) and Section 8 (Threats to Validity) are present with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: training data overlap (7/66 bugs), single-run experiments, manually examining patches for correctness, Completion Engine soundness depending on implementation, baseline results taken from prior work without reproduction."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 states specific boundaries: only Java evaluated, only two LLMs tested, only APR task (not other code generation tasks), difficulty with dynamically typed languages, limited evidence for rare token claim."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Zenodo artifact [62] and GitHub repo contain the full set of correct patches. Defects4J benchmark is publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes the data: Defects4J is a manually curated Java dataset. Bug selection criteria (single-hunk, compatible with Completion Engine) are stated with counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from a standard benchmark (Defects4J)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: start with Defects4J 1.2 (391 bugs - 4 deprecated), filter to single-hunk, remove incompatible = 138 bugs. For D4J 2.0: 438 bugs filtered to 135 single-hunk. Repair templates applied for RQ1/RQ2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section: 'This work was partially supported by NSF grants CCF-2131943 and CCF-2141474, as well as Kwai Inc.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors affiliated with University of Illinois Urbana-Champaign. No evaluated product is from their employer."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is an independent funder. Kwai Inc. does not have a direct stake in APR tool outcomes. Neither funder's product is being evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit training data cutoff date is stated for CodeT5 or InCoder. The paper acknowledges overlap risk but does not state when training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly addresses this: '7 out of 66 and 6 out of 50 overlap with training data on Defects4J 1.2 and 2.0 respectively' and provides adjusted results excluding these."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 addresses contamination: computes overlap between CodeT5 training data and Defects4J, reports adjusted results. For InCoder, acknowledges the threat cannot be fully addressed since training data is not revealed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Generation time per patch reported in Table 3 (e.g., 0.232s for base, 0.248s for full Repilot) and Table 4. Overhead percentages discussed (7% for CodeT5, negligible for InCoder)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified (Section 5.1): NVIDIA RTX A6000, 5-hour timeout per bug, 5000 or 500 samples per bug. Generation time per patch reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Repilot fixes 27% more bugs on Defects4J 1.2 and 47% more on Defects4J 2.0 than state-of-the-art baselines.",
       "evidence": "Table 1: Repilot fixes 66 bugs vs AlphaRepair's 52 on D4J 1.2 (27% more), and 50 vs 34 on D4J 2.0 (47% more).",
       "supported": "strong"
     },
     {
       "claim": "Repilot achieves ~60% compilation rate at 5000 patches, far exceeding prior tools.",
       "evidence": "Table 2: Repilot achieves 59% at Top-5000 vs AlphaRepair's 13% and CURE's 9%.",
       "supported": "strong"
     },
     {
       "claim": "All components (pruning, memorization, active completion) contribute positively to Repilot's effectiveness.",
       "evidence": "Table 3 ablation: compilation rate increases from 43.2% (base) to 60.7% (pruning) to 63.4% (full), with generation time overhead reduced from 25% to 7%.",
       "supported": "strong"
     },
     {
       "claim": "Repilot generalizes across different LLMs (CodeT5 and InCoder) and bug datasets.",
       "evidence": "Table 4: Repilot improves over baseline for both CodeT5 and InCoder on both D4J 1.2 and 2.0.",
       "supported": "moderate"
     },
     {
       "claim": "Repilot incurs minimal overhead (7% for CodeT5, negligible for InCoder).",
       "evidence": "Table 3: 0.232s vs 0.248s for CodeT5. Table 4: 1.70s vs 1.70s for InCoder.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Repilot fuses LLMs with semantics-based Completion Engines to guide autoregressive code generation during automated program repair. On Defects4J 1.2 and 2.0, Repilot fixes 66 and 50 single-hunk/single-line bugs respectively, outperforming 19 baseline APR tools. The Completion Engine pruning raises compilation rates from ~43% to ~63% with only 7% time overhead for CodeT5, and negligible overhead for the larger InCoder model. The approach generalizes across two LLM architectures (encoder-decoder and decoder-only) and two benchmark versions.",
   "red_flags": [
     {
       "flag": "Single-run experiments",
       "detail": "All experiments were run only once, as acknowledged in Section 8. With sampling-based generation (5000 or 500 samples), results could vary across runs, but no variance is reported."
     },
     {
       "flag": "Baseline results not reproduced",
       "detail": "Section 8 states: 'we follow the convention used in prior work to directly report the bug fix results without reproducing them.' This means baseline comparisons rely on numbers from other papers under potentially different conditions."
     },
     {
       "flag": "Limited language coverage",
       "detail": "Only Java is evaluated despite claims of generalizability. The paper acknowledges this but the title and framing suggest broader applicability."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational Codex paper establishing LLM code generation evaluation methodology."
     },
     {
       "title": "AlphaRepair: Zero-shot cloze-style APR using LLMs",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Direct predecessor and primary LLM baseline for automated program repair."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2305.01210",
       "relevance": "Evaluation methodology for LLM-generated code correctness."
     },
     {
       "title": "InCoder: A Generative Model for Code Infilling and Synthesis",
       "authors": ["Daniel Fried"],
       "year": 2023,
       "relevance": "One of two LLMs used in Repilot's evaluation; decoder-only code infilling model."
     },
     {
       "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "relevance": "Prior APR work that also uses static analysis to constrain code generation, key baseline."
     },
     {
       "title": "Patch Generation with Language Models: Feasibility and Scaling Behavior",
       "authors": ["Sophia D Kolak", "Ruben Martins", "Claire Le Goues", "Vincent Josua Hellendoorn"],
       "year": 2022,
       "relevance": "Studies feasibility and scaling of LLMs for patch generation in APR."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li"],
       "year": 2022,
       "doi": "10.1126/science.abq1158",
       "relevance": "Large-scale LLM code generation with sampling-based evaluation methodology."
     },
     {
       "title": "ChatRepair: Conversational APR approach",
       "authors": ["Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Dialogue-based LLM approach for APR using iterative feedback from test failures."
     },
     {
       "title": "Recoder: Edit-based NMT model for APR",
       "authors": ["Shangwen Wang"],
       "year": 2022,
       "relevance": "NMT-based APR that enforces syntax correctness via edit-based model and placeholder tokens."
     },
     {
       "title": "RewardRepair: Penalizing uncompilable patches during training for APR",
       "authors": ["Hao Ye"],
       "year": 2022,
       "relevance": "APR approach that boosts compilation rate through training-time penalties, key baseline for compilation rate comparison."
     }
   ]
 }