ai-research-survey

scan.json (26294B)

---

 {
   "paper": {
     "title": "Automated Test Case Repair Using Language Models",
     "authors": [
       "Ahmadreza Saboor Yaraghi",
       "Darren Holden",
       "Nafiseh Kahani",
       "Lionel Briand"
     ],
     "year": 2024,
     "venue": "IEEE Transactions on Software Engineering",
     "arxiv_id": "2401.06765",
     "doi": "10.1109/TSE.2025.3541166"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub link for TARGET (https://github.com/Ahmadreza-SY/TaRGet) in footnote 1 and states 'we provide the implementation of TARGET, TARBENCH, and the script for reproducing all experiments' (Section 5.1)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "TARBENCH is publicly available via Figshare at https://doi.org/10.6084/m9.figshare.25008893 (footnote 2). The benchmark comprising 45,373 broken test repairs is explicitly released."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.9 specifies the implementation uses Hugging Face Transformers and Accelerate libraries, with detailed hyperparameters. Hardware is specified: two Nvidia Quadro RTX 6000 GPUs (24GB each), Intel Xeon Gold 6234 16-Core CPU, 187GB RAM. Maven 3.6.3 and JDK versions 1.8.0_192, 11.0.16_8, or 17.0.2 are stated in Section 4.2.2."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we provide the implementation of TARGET, TARBENCH, and the script for reproducing all experiments' (Section 5.1, Table 11 'Reproducibility' column). The replication package is referenced via the GitHub repository."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Tables 5, 6, and 7 report only point estimates (e.g., 66.1% EM, 80.0% PR) without confidence intervals or error bars. No uncertainty quantification is provided for the primary evaluation metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "In RQ3.2, a paired t-test is used to compare Project-agnostic and Best models (p-value = 0.034), preceded by a Shapiro-Wilk normality test (p-values 0.16 and 0.25). This is appropriate for the comparative claim made."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "In RQ3.2, Cohen's d = 0.513 is reported as the effect size, described as 'moderate range, leaning closer to a small effect size.' Percentage point differences are consistently reported throughout (e.g., 37.4 EM pp improvement over NOCONTEXT)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or explicit justification for the sample sizes is provided. The benchmark size (45,373 instances) is argued to be larger than prior work but there is no formal analysis of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main experimental results (Tables 5, 6, 7) are from single fine-tuning runs. No standard deviation or variance across multiple runs with different random seeds is reported. The only standard deviations mentioned are for AST action counts in RQ3.2 analysis, not for model performance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "RQ1.2 compares TARGET against three baselines: CEPROT (a recent state-of-the-art method), SUTCOPY (a simple copy-based baseline), and NOCONTEXT (CLM without repair context). Results are reported in Tables 6 and 7."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "CEPROT is from ASE 2023, which is contemporary. The paper also justifies why other prior works (ReAssert, TRIP, etc.) could not be compared against due to reproducibility issues (Section 5.1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper systematically evaluates four IO formats (IO1-IO4) across three models (Table 5), effectively ablating the repair context selection, hunk prioritization, hunk representation, and output format. RQ3.1 ablates training data size. The trivial repair inclusion experiment (Section 4.3.3) also serves as an ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four evaluation metrics are used: Exact Match Accuracy (EM), Plausible Repair Accuracy (PR), BLEU, and CodeBLEU (Section 4.1). All four are reported in Tables 5 and 7."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5.3 describes a qualitative analysis where the authors 'manually analyzed many examples from different repair categories and projects, and selected eight representative and insightful cases.' Section 4.4.3 includes manual review of five comparisons between TARGET and CEPROT. These constitute human evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.5 describes temporal splitting: 'for each project, we kept older commits in the training set and included newer commits in both the validation and test sets.' The splits are 80% train, 5% validation, 15% test (36,639 / 1,631 / 7,103 instances)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "RQ2.1 provides detailed per-category breakdowns across repair categories (ARG, ORC, INV, etc.) and AST action counts. Figure 9 shows heatmaps of EM and PR across repair categories and complexity levels. Table 8 provides distribution across categories."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.5.3 and Appendix A present four detailed failure examples with analysis of why TARGET failed, including missing contextual information and complexity of multi-step repairs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "IO4 (Edit Sequence Output) is reported as performing worst across all models and metrics. The paper states 'IO4 reaching the lowest performance in all instances' (Section 4.3.3) and recommends against its use. Performance degradation with complex repairs is also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 66.1% EM accuracy (supported by Table 5), effectiveness across different repair scenarios (supported by RQ2.1), practical prediction guide (supported by RQ2.2), and generalizability to new projects (supported by RQ3.2). All claims are substantiated in the results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims are primarily through ablation-style comparisons: TARGET vs. NOCONTEXT demonstrates the effect of repair context (controlled single-variable manipulation). IO format comparisons (Table 5) isolate specific design choices. The data size experiment (RQ3.1) systematically varies one factor."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only on Java/JUnit projects but makes broad claims like 'our primary contributions... are not dependent on any specific programming language and can therefore be widely adopted' (Section 4.9). While they acknowledge the Java focus, the title 'Automated Test Case Repair Using Language Models' implies broader applicability than what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.10 discusses threats to validity, including the PR metric limitation where removing assertions could count as plausible repairs (with quantitative analysis showing 5.3 pp worst-case impact). The paper considers that PLB vs. CG performance differences may be due to 'pre-training datasets and tasks' not just model size (Section 4.3.3)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions with parameter counts are stated: CodeT5+ 770M, PLBART Base 140M, and CodeGen Multi 350M (Section 4.3.1). These are specific, identifiable open-source models with known architectures."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "This paper fine-tunes language models rather than using prompting. The input format is defined programmatically (Section 3.2) rather than through natural language prompts."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.9 reports: 4 epochs with early stopping, beam sizes (200 for PLB, 40 for CT5+ and CG), batch sizes (8 for PLB, 1 for CT5+, 2 for CG), Adam optimizer with weight decay (AdamW), learning rate 1e-5, cosine learning rate scheduler. Max input/output lengths of 512/256 tokens stated in Section 4.2.3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. TARGET is a fine-tuned model pipeline, not an agentic system with tool use, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.3 documents preprocessing in detail with counts at each stage: duplicates (1,039 excluded), empty repair context (3,748 excluded), absence of breakage location (1,274 excluded), max length exceeded (5,383 excluded). Section 4.2.4 documents quality checks with trivial repair removal (2,046 from eval, 7,196 kept in training) and test refactoring exclusion (6,302). Figure 7 provides a visual overview."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.10 'Threats to Validity' provides a dedicated discussion of external and internal validity threats, spanning approximately two pages of substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.10 discusses specific threats: (1) possibly simple test repairs and mitigation via trivial repair exclusion, (2) PR metric limitation with quantitative analysis showing 5.3 pp worst-case impact from 377 potentially misclassified assertion cases, (3) Java-only focus. These are specific to this study, not generic disclaimers."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states several scope boundaries: single-hunk repairs only (Section 4.2.1), Java/JUnit focus (Section 4.9), 512 token input limit (Section 4.2.3), breakage location assumed known (Section 3), and exclusion of complete file additions/removals from repair context (Section 4.2.2)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "TARBENCH is publicly available via Figshare (https://doi.org/10.6084/m9.figshare.25008893). The data is traceable to specific GitHub commits (project + commit hash pairs are included), enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the complete data collection procedure in extensive detail: starting from 4,662 CodeSearchNet Java projects, filtering to 1,257 Maven projects with 50+ stars, analyzing commits for test method changes, three-step validation execution, with counts at each filtering stage (Figure 7)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The data consists of open-source software project commits, and the project selection process is documented in Section 4.2.1."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 provides a visual pipeline from 1,257 projects through subject selection (198,210 tests from 116 projects), test repair collection (65,165 from 59 projects), preprocessing and filtering steps with counts, to final splits (36,639 train / 1,631 validation / 7,103 test). Each filtering criterion and the number of excluded instances are documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgement section lists funding: 'This work was supported by a research grant from Huawei Technologies Canada, Mitacs Canada, as well as the Canada Research Chair and Discovery Grant programs of NSERC and the Research Ireland grant 13/RC/2094-2.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Ottawa and Lero/University of Limerick (Saboor Yaraghi, Briand), Carleton University (Holden, Kahani). No authors are affiliated with the companies whose products are evaluated (the CLMs are open-source academic models)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders include Huawei Technologies Canada, Mitacs, NSERC, and Research Ireland. The paper evaluates open-source academic models (CodeT5+, PLBART, CodeGen), none of which are Huawei products. Huawei has no direct financial stake in the performance comparison between these specific models."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. While the paper has an acknowledgements section listing funding, there is no explicit declaration about patents, equity, or other financial interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses pre-trained models (CodeT5+, PLBART, CodeGen) but does not state the training data cutoff dates for any of them. Since these models are fine-tuned on TARBENCH data drawn from GitHub repositories, there is a risk that the pre-training data overlapped with the benchmark projects."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the pre-trained models may have seen the test repair code during pre-training. The temporal split (older commits for training, newer for testing) addresses the fine-tuning data split but not the pre-training data overlap. The APR section (5.2) briefly mentions that 'data leakage is a point of concern' for LM-based techniques per Jiang et al. [46], but the authors do not address this for their own work."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "TARBENCH is constructed from GitHub open-source projects. The CLMs (CodeT5+, PLBART, CodeGen) were pre-trained on code from GitHub and similar sources. The paper does not discuss whether TARBENCH projects' code appeared in the pre-training data of these models, despite this being a recognized concern in the related work section."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The evaluation is entirely automated using benchmark data from open-source repositories."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available open-source code from GitHub."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or per-example time is reported. The paper mentions beam sizes and hardware but does not quantify the wall-clock time or cost per repair."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the hardware is described (two Nvidia Quadro RTX 6000 GPUs, Section 4.9), the total GPU hours, training time, or computational budget is not quantified. Given that 12 model configurations were fine-tuned on 36k instances each, this represents significant compute that is not reported."
       }
     }
   },
   "claims": [
     {
       "claim": "TARGET achieves 66.1% exact match accuracy and 80% plausible repair accuracy using CodeT5+ 770M with IO2 format.",
       "evidence": "Table 5 (Section 4.3.3) shows CT5+ with IO2 achieves 66.1 EM and 80.0 PR, which are the best results across all model-IO combinations tested.",
       "supported": "strong"
     },
     {
       "claim": "TARGET outperforms the baseline (NOCONTEXT) by 37.4 EM percentage points.",
       "evidence": "Table 7 (Section 4.4.3) shows TARGET at 66.1 EM vs. NOCONTEXT at 28.7 EM, a difference of 37.4 pp. All four metrics show consistent improvement.",
       "supported": "strong"
     },
     {
       "claim": "TARGET significantly outperforms CEPROT, achieving 40.6% EM vs. CEPROT's 21% EM on CEPROT's benchmark.",
       "evidence": "Table 6 (Section 4.4.3) reports TARGET at 40.6 EM and 91.1 CodeBLEU vs. CEPROT at 21 EM and 60.4 CodeBLEU on 214 compatible instances from CEPROT's evaluation set.",
       "supported": "moderate"
     },
     {
       "claim": "PLBART 140M performs comparably to CodeT5+ 770M despite having far fewer parameters.",
       "evidence": "Table 5 shows PLB (IO2) at 57.9 EM vs. CT5+ at 66.1 EM, and PLB (IO3) at 59.1 EM. The PR gap is smaller: PLB at 79.2 vs. CT5+ at 80.0. The paper states losses of '8.2, 0.8, 1.4, and 0.8 pp' across metrics (Section 4.3.3).",
       "supported": "moderate"
     },
     {
       "claim": "TARGET can generalize to unseen projects with an average EM loss of only 4.87 percentage points.",
       "evidence": "Table 10 (RQ3.2) shows 10-fold results where project-agnostic models average 59.1% EM vs. best model varying per fold. A paired t-test shows statistical significance (p=0.034) but Cohen's d=0.513 indicates moderate effect size.",
       "supported": "strong"
     },
     {
       "claim": "A Random Forest model can predict whether TARGET will successfully repair a test with 88% F1 for exact match and 92% F1 for plausible repair.",
       "evidence": "Table 9 (Section 4.6.2) reports precision/recall/F1 of 87/88/88 for exact match and 90/94/92 for plausible, using 5-fold cross validation on the test set.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "TARGET, a language model-based approach for automated test case repair, achieves 66.1% exact match accuracy and 80% plausible repair accuracy on TARBENCH, a new benchmark of 45,373 broken test repairs from 59 Java projects. Fine-tuning CodeT5+ 770M with text-similarity-based hunk ordering outperforms both CEPROT and no-context baselines by large margins. The approach generalizes to unseen projects with only moderate performance loss (4.87 pp EM), and a Random Forest model can predict repair reliability with high accuracy.",
   "red_flags": [
     {
       "flag": "No variance across runs",
       "detail": "All main results appear to be from single fine-tuning runs without reporting variance across different random seeds. Deep learning results can vary significantly with initialization, and the absence of multi-run statistics makes it hard to assess result stability."
     },
     {
       "flag": "Pre-training contamination not addressed",
       "detail": "The CLMs (CodeT5+, PLBART, CodeGen) were pre-trained on GitHub code, and TARBENCH is derived from GitHub projects. The paper acknowledges this concern for APR work in Section 5.2 but does not address it for their own evaluation, despite the risk that pre-trained models may have memorized code patterns from the benchmark projects."
     },
     {
       "flag": "Selective baseline comparison with CEPROT",
       "detail": "The comparison with CEPROT uses only 214 out of 520 evaluation instances (41%) from CEPROT's test set. Multiple categories of instances were excluded (multi-hunk, no source changes, method name changes, invalid cases), which could introduce selection bias favoring TARGET's design constraints."
     },
     {
       "flag": "No training or inference time reported",
       "detail": "Despite fine-tuning 12+ model configurations on 36k instances with beam sizes up to 200, no training time, inference time, or total compute budget is reported. This makes it impossible to assess practical feasibility for adoption."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLM applications in software engineering, covering 50+ models and their SE task performance."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational work for evaluating code generation LLMs."
     },
     {
       "title": "CodeT5+: Open code large language models for code understanding and generation",
       "authors": ["Y. Wang", "H. Le", "A. D. Gotmare", "N. D. Q. Bui", "J. Li", "S. C. H. Hoi"],
       "year": 2023,
       "relevance": "The best-performing model in this study (770M parameters); encoder-decoder architecture for code tasks."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["N. Jiang", "K. Liu", "T. Lutellier", "L. Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Comparative study of CLMs vs. DL-based APR techniques showing LMs can surpass traditional approaches, directly relevant to evaluating LLMs for code repair."
     },
     {
       "title": "Identify and update test cases when production code changes: A transformer-based approach",
       "authors": ["X. Hu", "Z. Liu", "X. Xia", "Z. Liu", "T. Xu", "X. Yang"],
       "year": 2023,
       "relevance": "CEPROT baseline compared against in this paper; addresses automated detection and updating of obsolete tests using CodeT5."
     },
     {
       "title": "A survey of learning-based automated program repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Ma", "W. Sun", "Z. Chen"],
       "year": 2023,
       "arxiv_id": "2301.03270",
       "relevance": "Survey of learning-based APR approaches, closely related to the test repair task addressed in this paper."
     },
     {
       "title": "Unified pre-training for program understanding and generation",
       "authors": ["W. Ahmad", "S. Chakraborty", "B. Ray", "K.-W. Chang"],
       "year": 2021,
       "relevance": "PLBART model used as one of the three CLMs in this study; pre-trained on Java and Python."
     },
     {
       "title": "CodeGen: An open large language model for code with multi-turn program synthesis",
       "authors": ["E. Nijkamp"],
       "year": 2023,
       "relevance": "CodeGen model used as one of the three CLMs evaluated in this study; decoder-only architecture."
     },
     {
       "title": "Intent-preserving test repair",
       "authors": ["X. Li", "M. d'Amorim", "A. Orso"],
       "year": 2019,
       "relevance": "TRIP: prior work on test repair using dynamic symbolic execution, evaluated on 91 tests. Key baseline for comparison despite reproducibility issues."
     },
     {
       "title": "Iter: Iterative neural repair for multi-location patches",
       "authors": ["H. Ye", "M. Monperrus"],
       "year": 2024,
       "relevance": "Iterative neural program repair technique supporting multi-hunk repairs; cited as promising future direction for extending TARGET."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair: zero-shot APR approach that frames program repair as code generation without fine-tuning on repair datasets."
     }
   ]
 }