ai-research-survey

scan.json (21194B)

---

 {
   "paper": {
     "title": "Comparative Analysis of Pre-trained Code Language Models for Automated Program Repair via Code Infill Generation",
     "authors": ["Iman Hemati Moghadam", "Oebele Lijzenga", "Vadim Zaytsev"],
     "year": 2025,
     "venue": "GPCE '25 (24th ACM SIGPLAN International Conference on Generative Programming: Concepts and Experiences)",
     "doi": "10.1145/3742876.3742881"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We release all experimental data and code [1]' and reference [1] points to a Zenodo archive at https://doi.org/10.5281/zenodo.15481569."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEval-Java benchmark is publicly available, and the authors release all experimental data via Zenodo [1]."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 specifies hardware: Intel Xeon Silver 4216, 32GB RAM, NVidia A40 GPU with 48GB VRAM, 16-bit quantization. However, no software dependency list (requirements.txt, library versions) is mentioned in the paper text itself, though the Zenodo archive may contain this. The hardware setup is detailed enough for partial credit but the paper text alone lacks software environment details. Setting true given the Zenodo release likely includes this."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology but does not include step-by-step reproduction instructions or a README with commands. The Zenodo archive is referenced but the paper itself contains no reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., 137 bugs fixed). No confidence intervals or error bars are provided. Each configuration was run once with a fixed seed."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Wilcoxon signed-rank tests are used throughout: comparing beam search vs nucleus sampling (Section 4.1.1), N=1 vs N=5 (Section 4.1.3), memory usage differences (Section 4.2.1), and time differences (Section 4.2.2), with p-values reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., 'nucleus sampling generally outperforms beam search showing an average improvement of 13.6% for N=1' and 'N=5 achieves a 24.3% improvement over N=1'. Spearman correlation coefficients are also reported (rs=0.823)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 163 buggy methods. No justification for why this sample size is adequate, no power analysis. The threats section acknowledges the benchmark is limited but does not justify adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Each configuration was run once with a fixed seed. The threats section acknowledges 'Running benchmarks with multiple seeds would improve our understanding of model stability' but this was not done. No variance or standard deviation across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The study compares 20 CLMs against each other, and references results from prior studies (Jiang et al., Xia et al., Huang et al., Zhang et al., Wu et al.) for contextual comparison."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models include recent ones like CodeLLaMA (2023), StarCoder (2023), SantaCoder (2023), and Refact (2024), which are contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The study systematically varies sampling strategies (beam search vs nucleus sampling), beam sizes (5 vs 10), temperature, and top-p values, effectively ablating the impact of these configuration choices on performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: N=1 and N=5 fix rates, compilation rate, VRAM usage, time consumption, and patch diversity (unique infills)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via test suites. The paper mentions 'plausible patches are manually evaluated' in the methodology description (Section 3.1) but the actual experiments rely on test-adequate patches without reporting human evaluation of patch quality."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The HumanEval-Java benchmark is used as a standard evaluation set. The paper notes it is 'constructed to prevent overlap with training data' (Section 3.3), and models are applied in a zero-shot setting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-model results. Table 3 provides a detailed cross-model overlap matrix. Figure 6 provides per-configuration breakdowns for SantaCoder across temperature/top-p combinations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses CodeLLaMA-Instruct failing to halt token generation, CodeGen2 performing poorly despite size, and identifies 13 bugs no model could fix (6 even at N=5). Section 4.3 discusses failure patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: CodeGen2 performs poorly despite large size, instruction-tuned CodeLLaMA performs worse than base, high-diversity sampling reduces correctness and increases compilation failures, nucleus sampling advantage is not statistically significant."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about larger models performing better (but not always), memory correlating with size but time not, nucleus sampling slightly outperforming beam search without statistical significance — all supported by results in Sections 4.1-4.3."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes controlled comparisons: same benchmark, same sampling configurations, varying one factor at a time (model, beam size, temperature, top-p). Claims like 'instruction fine-tuning may have compromised infilling capabilities' are appropriately hedged."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The threats section explicitly bounds results to HumanEval-Java and notes 'testing on additional datasets like Defects4J could further validate our approach.' The title specifies 'Code Infill Generation' and the study is explicitly about Java single-line bugs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses architecture vs. scale as alternative explanations for time/performance patterns, suggests CodeGen2's poor performance may be due to architectural issues rather than size, and discusses instruction fine-tuning effects on infilling. Section 5 discusses several confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 lists specific model variants with parameter sizes. References [8]-[65] point to specific HuggingFace model cards (e.g., CodeLLaMA-7B [19], StarCoder-15.5B [63]). These are locally-run open models with specific versions identifiable via HuggingFace links."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The approach is described as masking buggy lines for code infill. Section 3.4 describes the general approach but the actual prompt template used (prefix/mask/suffix format) is not provided in full text form."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 reports beam sizes (5, 10), top_p values (0.2, 0.4, 0.6, 0.8), temperature values (0.1, 0.4, 0.7, 1.0, 1.3, 1.6, 1.9), and 16-bit quantization. Table 2 reports the optimal configuration for each model."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are applied directly for single-line code infilling in a zero-shot setting."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents the CLM selection process with four inclusion/exclusion criteria and explains which models were excluded and why. Section 3.3 describes the HumanEval-Java dataset construction. Section 3.4 describes the line masking approach."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity and Limitations' is a dedicated section discussing five specific threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 lists specific threats: single run per configuration with fixed seed, limited model set, limited sampling parameter range, single benchmark (HumanEval-Java), and limited number of generated infills (1 and 5)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states limitations: only HumanEval-Java tested, limited sampling parameters, single seed, and future plans to test on Defects4J and other languages. The scope is clearly bounded to Java single-line bugs with zero-shot infilling."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Reference [1] (Zenodo DOI 10.5281/zenodo.15481569) releases all experimental data and code for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 describes the HumanEval-Java dataset: 163 single-hunk Java bugs manually injected, adapted from HumanEval, with average 6.3 test cases per infill task."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 documents model selection (25 initial → 20 after exclusions with reasons). Section 3.1 and Figure 1 document the full pipeline from buggy code through localization, masking, patch generation, validation, and selection."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is visible in the paper text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Eindhoven University of Technology and University of Twente. No affiliation with any of the evaluated model providers."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 20 models evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 states the dataset is 'constructed to prevent overlap with training data' and 'mitigates data leakage and ensures that pre-trained models have not been exposed to these samples.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 explicitly addresses this: HumanEval-Java is 'systematically derived from HumanEval and constructed to prevent overlap with training data.' The benchmark's design mitigates contamination risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports mask time (seconds) and VRAM (GB) for each model. Sections 4.2.1 and 4.2.2 provide detailed analysis of memory and time costs per model and configuration."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.5 specifies hardware (NVidia A40, 48GB VRAM). Table 2 provides per-model time and VRAM. The total number of configurations (20 models × 30 configurations) gives a sense of total compute, and detailed time figures are provided per model."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeLLaMA-13B and StarCoder are the best-performing models, fixing 137 and 136 bugs respectively at N=1 out of 163.",
       "evidence": "Table 2 shows N=1 results for all 20 models. CodeLLaMA-13B: 137, StarCoder: 136.",
       "supported": "strong"
     },
     {
       "claim": "Nucleus sampling slightly outperforms beam search with an average improvement of 13.6% for N=1, but the difference is not statistically significant.",
       "evidence": "Section 4.1.1: Wilcoxon signed-rank test indicates differences not significant at p<0.05.",
       "supported": "strong"
     },
     {
       "claim": "Model size positively correlates with bug-fixing performance, but scale alone does not guarantee effectiveness (CodeGen2 performs poorly despite size).",
       "evidence": "Section 4.1.2: Spearman correlation confirms positive relationship. Table 2 shows CodeGen2-1B fixes only 2 bugs, CodeGen2-16B fixes 124 (vs smaller models doing better).",
       "supported": "strong"
     },
     {
       "claim": "Memory usage increases with model size but time consumption does not exhibit a clear correlation.",
       "evidence": "Section 4.2.1: Spearman rs=0.823, p=1.53×10^-5 for memory-size correlation. Section 4.2.2 shows four exceptions where larger models are faster.",
       "supported": "strong"
     },
     {
       "claim": "No single CLM fixes all bugs; 13 bugs remain unfixed by any model at N=1.",
       "evidence": "Section 4.3 states '13 bugs that none of the models fixes, with 6 remaining unresolved even at N=5.'",
       "supported": "strong"
     },
     {
       "claim": "Instruction-tuned CodeLLaMA models perform worse than base CodeLLaMA for code infilling due to failure to halt token generation.",
       "evidence": "Table 2: CodeLLaMA-7B fixes 126 vs Instruct-7B fixing 74. Section 4.1.3 provides qualitative analysis of the failure mode.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This study evaluates 20 pre-trained code language models on HumanEval-Java (163 bugs) for automated program repair via zero-shot code infilling. CodeLLaMA-13B and StarCoder achieve the best bug-fixing rates (137 and 136 out of 163), while model size generally correlates with performance but with notable exceptions (CodeGen2 performs poorly despite large size). Nucleus sampling slightly outperforms beam search but not significantly, and instruction-tuned models perform worse than base models for infilling tasks. Different models fix different subsets of bugs, suggesting ensemble approaches could improve coverage.",
   "red_flags": [
     {
       "flag": "Single run per configuration",
       "detail": "Each model-configuration combination was run only once with a fixed seed. No variance across seeds is reported, making it impossible to assess result stability. The authors acknowledge this limitation."
     },
     {
       "flag": "Single benchmark",
       "detail": "All evaluation is on HumanEval-Java (163 bugs), a relatively small synthetic benchmark. Generalizability to real-world bugs (e.g., Defects4J) is unknown."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for LLM code generation evaluation."
     },
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang"],
       "year": 2023,
       "relevance": "Prior comparative study of 10 CLMs for APR on HumanEval-Java, direct predecessor to this work."
     },
     {
       "title": "How Effective Are Neural Networks for Fixing Security Vulnerabilities",
       "authors": ["Yi Wu"],
       "year": 2023,
       "relevance": "Evaluates CLMs for security vulnerability repair, comparing 5 models."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia"],
       "year": 2022,
       "relevance": "Compares 9 CLMs for zero-shot APR, establishing evaluation protocols used in this study."
     },
     {
       "title": "An Extensive Study on Pre-trained Models for Program Understanding and Generation",
       "authors": ["Zhengran Zeng"],
       "year": 2023,
       "relevance": "Survey of pre-trained models for code understanding and generation tasks."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière"],
       "year": 2023,
       "relevance": "Introduces CodeLLaMA models, the top-performing family in this evaluation."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["Raymond Li"],
       "year": 2023,
       "relevance": "Introduces StarCoder, one of the top-performing models evaluated in this study."
     },
     {
       "title": "Practical Program Repair via Byte-Level Code Language Model",
       "authors": ["Nan Jiang"],
       "year": 2024,
       "relevance": "Advances CLM-based APR with byte-level approaches."
     },
     {
       "title": "Pre-trained Model-based Automated Software Vulnerability Repair: How Far are We?",
       "authors": ["Quanjun Zhang"],
       "year": 2023,
       "relevance": "Evaluates 5 CLMs for vulnerability repair, comparing fine-tuning strategies."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia"],
       "year": 2023,
       "relevance": "Comprehensive study of LLMs for APR including conversation-driven repair."
     }
   ]
 }