ai-research-survey

scan.json (18404B)

---

 {
   "paper": {
     "title": "Can We Automatically Fix Bugs by Learning Edit Operations?",
     "authors": ["Aidan Connor", "Aaron Harris", "Nathan Cooper", "Denys Poshyvanyk"],
     "year": 2022,
     "venue": "IEEE/ACM International Conference on Automated Software Engineering (ASE)",
     "doi": null
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/WM-SEMERU/hephaestus (Section IX/Conclusions)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses the publicly available Bugs2Fix subset from Microsoft's CodeXGlue project [11], a standard public benchmark."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an NVIDIA Titan RTX GPU and OpenNMT toolkit but does not provide a requirements.txt, Dockerfile, or detailed dependency list with versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section IX states 'replication documentation can be found at' the GitHub repository URL."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates only (e.g., PPA of 14.7%, 8.3%). No confidence intervals or error bars are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims differences are or are not 'significant' (e.g., 'no significant difference in PPA') but uses no statistical tests to support these claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw percentage differences are reported but no formal effect sizes (Cohen's d, etc.) are computed. The differences are described informally (e.g., 'approximately 0.8%')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains ~58,000 BFPs and 10% is used for testing, but there is no justification for why this size is adequate or any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported. Results appear to be from single training runs with no repeated experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "A control model (direct buggy-to-fixed translation) serves as the baseline, trained with the same parameters and data. This follows the approach of Tufano et al. [17]."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The control baseline replicates the approach from Tufano et al. [17] (2019) and the study also references CURE [7] (2021). These were contemporary at submission time."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies condensing strategy (basic, strict, loose), machine string form (typed vs general), and architecture (LSTM vs GRU), which functions as an ablation across design choices."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Perfect Prediction Accuracy, Failed Prediction Rate, Edit Distance Decrease, and Training Accuracy (Section V-A)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated fixes was performed. All evaluation is automated (exact match, edit distance)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D states: 'Each model is trained with 80% of their respective dataset and validated with 10%... We then test each Hephaestus model using the remaining 10%.'"
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by condensing strategy (basic, strict, loose), parameter group (LSTM+General, GRU+General, LSTM+Typed) in Figures 3a-3d."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-E (RQ1) discusses failure cases: 'the main cause of prediction failure was that some generated edit operations modified token indices that were out-of-bounds.'"
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper reports a negative result: edit operations-based models performed worse than the baseline. The paper honestly concludes 'the introduction of these specific methods... did not provide a benefit.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that 'all models which learned from edit operations were not as effective at repairing bugs as models which learned from fixed code segments directly,' which is supported by the results in Section VI."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim is that edit operations introduce more complexity leading to worse performance. The controlled experimental design (same data, same architecture, only varying the output representation) adequately supports this via single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VII (External Validity) explicitly states: 'we can only say that our method is ineffective for Java-based examples' and acknowledges the dataset limitation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses that higher entropy from the more complex output format is a likely explanation (Section VI-E, RQ1), and the Threats to Validity section discusses construct validity concerns about the edit operation representation choice."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using OpenNMT and LSTM/GRU architectures but does not specify the exact version of OpenNMT used."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper uses NMT models trained from scratch, not prompted language models. Prompting is not applicable."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: 2-layer LSTM/GRU, dropout rate 0.2, SGD optimizer with cross-entropy loss, 50,000 training steps, 80/10/10 train/val/test split. Parameters follow Tufano et al. [17]."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a standard NMT training and evaluation pipeline."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes dataset construction in detail: the CodeXGlue Bugs2Fix subset, abstraction process from Tufano et al. [17], filtering to 1-50 token methods, and the edit operation extraction pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Threats to Validity' provides a dedicated discussion of construct, external, and internal validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: the edit operation format choice (construct), Java-only dataset (external), and code abstraction limiting which BFPs can be used (internal)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII (External Validity): 'we can only say that our method is ineffective for Java-based examples.' The paper also notes limitation to methods with 1-50 tokens and the abstraction constraint."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The underlying Bugs2Fix/CodeXGlue dataset is publicly available [11], and the code repository is released, enabling verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C describes using the Bugs2Fix subset from CodeXGlue: ~58,000 Java method BFPs with 1-50 tokens, preformatted per Tufano et al. [17]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from a standard public benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from raw BFPs to edit operation extraction to dataset construction is documented in Section IV (Levenshtein operations, condensing algorithms, machine string encoding, dataset formatting in Table I)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section X: 'The authors have been supported in part by the NSF CCF-1955853 and CCF-2007246 grants.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with William and Mary, clearly listed on the first page. No product being evaluated is tied to their affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF is a government funding agency with no financial stake in whether edit operations work for bug repair."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper trains NMT models from scratch on the provided dataset. There is no pre-trained model whose training cutoff is relevant."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Models are trained from scratch with explicit 80/10/10 splits. No pre-trained model is evaluated on a benchmark."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No pre-trained language model is being evaluated. Models are trained from scratch on the dataset."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported for the models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-D: 'The training process took about 5.5 hours for each model using an NVIDIA Titan RTX GPU.' With 12 models total, this gives a clear compute budget."
       }
     }
   },
   "claims": [
     {
       "claim": "All models which learned from edit operations were not as effective at repairing bugs as models which learned from fixed code segments directly.",
       "evidence": "Control model PPA of 14.7% vs best experimental model PPA of 8.3% (Figure 3a). Control had better EDD (-1.32 avg vs -2.54 for basic, -1.57 strict, -1.61 loose) (Figure 3c).",
       "supported": "strong"
     },
     {
       "claim": "Condensing edit operations into strict and loose forms is beneficial over not condensing them at all.",
       "evidence": "EDD values: strict/loose models averaged ~-1.57/-1.61 vs basic models at -2.54 (Section VI-E, RQ2). PPA differences were negligible.",
       "supported": "moderate"
     },
     {
       "claim": "There is no significant difference in performance between LSTM and GRU architectures for this task.",
       "evidence": "PPA variation between LSTM and GRU is approximately 0.8-1% across conditions (Figure 3a, Section VI-E RQ3). No statistical test performed.",
       "supported": "weak"
     },
     {
       "claim": "The main cause of prediction failure in edit operation models was out-of-bounds token indices.",
       "evidence": "Stated in Section VI-E (RQ1) based on failure analysis, with reference to source documentation.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Hephaestus, a novel approach using Levenshtein edit operations for NMT-based automated bug repair, performed consistently worse than the standard direct buggy-to-fixed translation approach across all metrics. The best edit operations model achieved only 8.3% perfect prediction accuracy compared to 14.7% for the control. Condensing edit operations (strict/loose) provided marginal improvements over basic operations but still underperformed the baseline. The study is notable for honestly reporting negative results.",
   "red_flags": [
     {
       "flag": "No statistical tests for claimed differences",
       "detail": "The paper uses words like 'significant' and 'negligible' to describe differences between models without any statistical testing. All comparisons are based on eyeballing point estimates from single runs."
     },
     {
       "flag": "Single-run experiments",
       "detail": "Results appear to be from single training runs with no repeated experiments, making it impossible to assess whether observed differences are due to random variation in training."
     }
   ],
   "cited_papers": [
     {
       "title": "CODIT: Code Editing with Tree-Based Neural Machine Translation",
       "authors": ["Saikat Chakraborty", "Miltiadis Allamanis", "Baishakhi Ray"],
       "year": 2019,
       "doi": "10.1109/TSE.2020.3020502",
       "relevance": "Directly related approach using AST-level edit operations for code transformation via NMT."
     },
     {
       "title": "SEQUENCER: Sequence-to-Sequence Learning for End-to-End Program Repair",
       "authors": ["Zimin Chen"],
       "year": 2019,
       "doi": "10.1109/TSE.2019.2940179",
       "relevance": "Seq2seq approach to automated program repair, a key baseline in the APR literature."
     },
     {
       "title": "CURE: Code-Aware Neural Machine Translation for Automatic Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Lin Tan"],
       "year": 2021,
       "doi": "10.1109/icse43902.2021.00107",
       "relevance": "State-of-the-art NMT-based program repair with code-aware pre-training, a key contemporary baseline."
     },
     {
       "title": "An Empirical Study on Learning Bug-Fixing Patches in the Wild via Neural Machine Translation",
       "authors": ["Michele Tufano"],
       "year": 2019,
       "doi": "10.1145/3340544",
       "relevance": "Foundational NMT-based bug repair work whose methodology and dataset this paper builds on directly."
     },
     {
       "title": "On Learning Meaningful Code Changes via Neural Machine Translation",
       "authors": ["Michele Tufano"],
       "year": 2019,
       "doi": "10.1109/ICSE.2019.00021",
       "relevance": "Related work on learning code changes via NMT, relevant to automated program repair methodology."
     },
     {
       "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["Shuai Lu"],
       "year": 2021,
       "arxiv_id": "2102.04664",
       "relevance": "Source of the Bugs2Fix benchmark dataset used in this study, a widely-used code generation benchmark."
     },
     {
       "title": "Learning to Fix Build Errors with Graph2Diff Neural Networks",
       "authors": ["Daniel Tarlow"],
       "year": 2019,
       "arxiv_id": "1911.01205",
       "relevance": "Related approach using graph neural networks to predict code diffs for automated repair."
     },
     {
       "title": "Toward Better Evolutionary Program Repair: An Integrated Approach",
       "authors": ["Yuan Yuan", "Wolfgang Banzhaf"],
       "year": 2020,
       "doi": "10.1145/3360004",
       "relevance": "Evolutionary approach to automated program repair, representing an alternative paradigm to NMT-based methods."
     }
   ]
 }