ai-research-survey

scan.json (31714B)

---

 {
   "paper": {
     "title": "LLM Test Generation via Iterative Hybrid Program Analysis",
     "authors": ["Sijia Gu", "Noor Nashid", "Ali Mesbah"],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2503.13580",
     "doi": "10.1145/3744916.3764553"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Panta combines static control flow analysis with dynamic code coverage to iteratively guide LLMs in generating unit tests targeting uncovered execution paths. On 130 complex Java classes from Defects4J, Panta achieves 70.2% line coverage and 60.8% branch coverage with Llama 3.3 70B, outperforming a re-implemented SymPrompt baseline by 26 and 23 percentage points respectively. The ablation study shows the iterative framework contributes ~28pp coverage improvement over single-pass, while hybrid program analysis adds another ~10pp. Claude 3.5 Haiku performs best among four evaluated models, but mutation scores remain below 50% across all models, indicating LLM-generated tests still have weak fault detection.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Panta is publicly available at a GitHub repository (reference [25]: https://github.com/PANTA-TestAutomation/Panta). Section 6 states: 'we have made Panta's implementation and dataset publicly available.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available Defects4J benchmark (v2.0.1). Section 6 states implementation and dataset are publicly available at [25]. Section 4.1 notes 'More details about individual classes are publicly available in our repository.'"
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 specifies Java 8+, Maven v3.6.3, JUnit v4.13.2, JaCoCo v0.8.11, and Pitest v1.17.0. Combined with the public repository, this provides sufficient dependency information to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Reproducibility) states: 'we provide detailed instructions for replicating our experimental results' and points to the public repository at [25]."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates for all metrics (line coverage, branch coverage, pass rate, mutation score). No confidence intervals, error bars, or ± notation are provided anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "A Paired T-test is used in Section 4.3 (ablation) to show Panta_basic and Panta_cov are NOT significantly different. However, no significance tests are applied to the main claims: the 26%/23% improvement over SymPrompt (RQ1), or the model ranking (RQ3). The core outperformance claims are based on comparing numbers without any statistical test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage point differences are reported with baseline context throughout: '26.3% higher line coverage' with both values visible (70.18% vs 43.91% in Table 2). The ablation (Table 3) and model comparison (Table 4) similarly report differences with baseline values in the tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 130 classes and 2,971 MUTs are selected by criteria (CYC > 10, non-abstract, public), but there is no justification for why this sample size is sufficient for the claims, no power analysis, and no acknowledgment of whether the sample is adequate."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 explicitly states: 'Given the low temperature, the generation process is largely deterministic, with only minor fluctuations. As a result, we perform a single run per class.' No variance, standard deviation, or spread measure is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Panta is compared against SymPrompt (re-implemented from Ryan et al. 2024) in RQ1 (Table 2). The ablation study (RQ2) includes a single-pass baseline, Panta_basic, and Panta_cov variants."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "SymPrompt (2024) is described as a state-of-the-art LLM-based test generation technique. The paper also discusses CoverUp (2024) and HITS (2024) as recent related work, noting HITS could not be replicated."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 4.3, Table 3) presents a thorough ablation with four variants: single-pass baseline, Panta_basic (iterative framework only), Panta_cov (+ coverage info), and full Panta (+ path selection). This isolates each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five evaluation metrics are used: line coverage, branch coverage, pass rate, mutation score, and high coverage count (HCC). This provides a multi-faceted view of test quality."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated (coverage measurement, mutation testing, pass rate). No human evaluation of test quality, readability, or usefulness is conducted, despite the paper citing readability as a motivation for LLM-based test generation over search-based approaches."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no discussion of separating development data from evaluation data. The prompt templates and algorithm parameters (e.g., maxNoIncreaseLimit, numOfAttempt=3) may have been tuned on the same Defects4J projects used for evaluation. No explicit dev/test split is described."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per project in Tables 2, 3, and 4 (14 projects). Table 3 additionally breaks results by complexity class (CYC_max ≤ 20 vs > 20). Individual class details are available in the repository."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Limitations) discusses several failure modes: classes with heavy inheritance, LLMs' inability to fix runtime errors (AssertionError), low mutation scores, and difficulty with large utility classes. Section 4.2 discusses cases where SymPrompt outperforms Panta (Collections pass rate, JDatabind/Lang mutation score)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SymPrompt achieves higher pass rate in Collections and higher mutation score in JDatabind and Lang (Section 4.2). The ablation shows Panta_cov sometimes performs slightly worse than Panta_basic. Average mutation score is below 50% for all models."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '26% higher line coverage and 23% higher branch coverage compared to the state-of-the-art.' Table 2 shows 26.3pp higher line coverage (70.18% vs 43.91%) and 22.7pp higher branch coverage (60.83% vs 38.17%), matching the abstract."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'our iterative framework significantly enhances overall code coverage' and 'integrating code coverage-driven path selection further improves coverage by approximately 10%' are supported by the ablation study (Table 3), which uses controlled single-variable manipulation across four variants."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (External Validity) explicitly states: 'Our current implementation and evaluation focus on Java, which may impact the generalizability of our findings.' Section 5 (Application) bounds scope to Java SE projects with similar structural complexity. Though the title is broad, the paper body properly bounds claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses several alternative explanations: data contamination (LLMs may have seen Defects4J code), Comex tool limitations affecting path extraction, model capability as a confound, and coverage vs fault detection distinction. These are specific to the study."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures coverage and mutation score and claims these indicate test quality. They explicitly acknowledge the gap: 'coverage alone does not ensure fault detection if test assertions are non-existent or weak' (Section 4.1), and add mutation score specifically to address this proxy gap. Claims are stated at the measurement level (coverage, mutation score) without inflating to broader quality claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Llama 3.3 70B', 'Mistral Large 2', 'GPT-4o Mini', and 'Claude 3.5 Haiku'. These are marketing names without specific version strings, snapshot dates, or API version identifiers. No model checkpoint dates or API versions are stated."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows a prompt template with placeholders ({source_file_numbered}, {test_file}, {selected_path_for_method_1}, etc.), not the full prompt text. The test repair prompt is not shown at all. Any system-level instructions or output format requirements are not shown. While the repository may contain full prompts, the paper itself only provides the template structure."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports: 'max_tokens set to 4096 for output generation and temperature set to 0.2' for all four LLMs. Algorithm parameters are also stated: numOfAttempt=3 for repair (Section 3.4), and the stopping conditions (maxCYC iterations, maxNoIncreaseLimit)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative workflow is described in detail: Algorithms 1 (path extraction), 2 (path selection), and 3 (iterative framework) provide full pseudocode. Figure 2 shows the system overview. The test validation, repair loop, feedback mechanism, and stopping conditions are all documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 documents the class selection pipeline: starting from 14 Defects4J v2.0.1 projects, excluding 3 outdated projects, selecting non-abstract public classes with at least one method with CYC > 10, then manually excluding outliers (CYC > 40 switch patterns, inheritance-heavy classes). Final counts: 130 classes, 2,971 MUTs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Discussion) contains a dedicated 'Limitations' subsection discussing source code context, test repair, and test effectiveness limitations. Section 6 (Threats to Validity) discusses internal and external validity threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed include: Comex bugs potentially affecting path extraction (mitigated by verification during implementation), data contamination with Defects4J, Java-only evaluation limiting generalizability, large utility class challenges, and LLMs' inability to fix runtime errors like AssertionError."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states Panta 'is most applicable to Java SE projects with similar structural complexity' and that extending to Java EE 'remains an important direction for future work.' Section 6 states the Java focus 'may impact the generalizability.' They note the absence of cross-class context."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states 'More details about individual classes are publicly available in our repository.' Section 6 says 'we have made Panta's implementation and dataset publicly available.' The public repository at [25] contains per-class results and the evaluation dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the subject selection in detail: Defects4J v2.0.1 latest fixed versions, 14 projects after excluding 3 outdated ones, class selection criteria (non-abstract, public, CYC > 10), manual exclusion of outliers. Table 1 provides full details per project."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the standard Defects4J benchmark, a well-known public dataset in software testing research."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from subject selection through evaluation is documented: select Defects4J projects → filter classes by criteria → extract CFGs → run iterative test generation → validate tests → measure coverage and mutation score. Each step is described in Sections 3-4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section, acknowledgments, or grant information appears in the paper. Whether the research was funded is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with the University of British Columbia, clearly stated in the paper header. They are not affiliated with any of the LLM providers evaluated (Meta, Mistral, OpenAI, Anthropic)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of the funder cannot be assessed. Absence of funding disclosure is not the same as absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, patent disclosures, or financial interest declarations appear anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four evaluated models (Llama 3.3 70B, Mistral Large 2, GPT-4o Mini, Claude 3.5 Haiku). This is relevant because Defects4J has been public since 2014."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 (Internal Validity) explicitly discusses: 'data contamination, i.e., the possibility that LLMs may have encountered parts of the Defects4J codebase during pretraining.' They mitigate by comparing relative improvements between prompting strategies using the same model."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 acknowledges Defects4J code may be in training data and provides a mitigation strategy: 'all evaluated prompting strategies use the same set of projects and the same underlying LLM. Our evaluation therefore reports relative improvements between prompting strategies rather than absolute performance.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates an automated tool on open-source code."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions assigned to humans."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Runtime) reports wall-clock time: 'the average runtime for Panta to generate a passing test suite is 2.3 minutes per method, or approximately 53 minutes per class.' Per-project breakdowns are also provided (e.g., Cli: 0.8 min/method, Collections: 9.4 min/method). No dollar costs or token counts are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-method and per-class runtimes are stated, but the total computational budget (total API spend, total wall-clock time for all experiments, or hardware used) is not quantified. For 2,971 methods × 4 models, this represents substantial compute that is not totaled."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 explicitly states: 'we perform a single run per class.' No seed sensitivity analysis is conducted despite using temperature=0.2 which still introduces stochasticity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 explicitly states: 'we perform a single run per class and report the average results across all classes for each project.' The number of runs (one) is clearly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Key parameters (temperature=0.2, max_tokens=4096, numOfAttempt=3, maxNoIncreaseLimit) are stated but no hyperparameter search is described. How these values were selected is not discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The selection of temperature=0.2, maxNoIncreaseLimit, and other parameters is not justified. The paper states these values without explaining why they were chosen or whether alternatives were considered."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 14 projects, 4 models, and multiple metrics. A paired t-test is used in one comparison (Section 4.3) but no correction for multiple comparisons (Bonferroni, Holm, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The main baseline (SymPrompt) was re-implemented by the authors because 'neither HITS nor SymPrompt has publicly available implementations.' The bias of evaluating their own system against their own re-implementation of the baseline is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Panta is iterative (multiple LLM calls per class, up to maxCYC iterations) while SymPrompt is single-pass. This substantial compute difference is not discussed or controlled for. No compute-matched comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Defects4J and measures coverage and mutation score as proxies for test quality. Whether coverage/mutation score on Defects4J classes with CYC > 10 is representative of real-world test generation challenges is not discussed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "In RQ3 (Table 4), four different LLMs are compared using the same Panta scaffold, properly controlling for the scaffold variable. In RQ1, the scaffold IS the independent variable being evaluated (Panta vs SymPrompt), so there is no confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses that LLMs may have seen Defects4J code during pretraining (Defects4J published 2014, all models trained after). They mitigate by comparing relative improvements between strategies rather than absolute performance."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage. For example, whether providing full source code and coverage reports in the prompt mirrors realistic development scenarios is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 130 classes come from only 14 projects. Classes within the same project share code patterns, dependencies, and coding styles. Non-independence across classes within projects is not discussed or addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination pipeline). The only mitigation is the relative comparison strategy."
       }
     }
   },
   "claims": [
     {
       "claim": "Panta achieves 26% higher line coverage and 23% higher branch coverage compared to the state-of-the-art (SymPrompt)",
       "evidence": "Table 2 shows average line coverage of 70.18% (Panta) vs 43.91% (SymPrompt) and branch coverage of 60.83% vs 38.17% across 130 classes from 14 Defects4J projects. Panta outperforms on all 14 projects for both metrics.",
       "supported": "moderate"
     },
     {
       "claim": "The iterative framework alone (Panta_basic) improves line coverage by 27.65% and branch coverage by 25.77% over a single-pass baseline",
       "evidence": "Table 3 ablation study shows baseline at 32.69% line / 24.87% branch coverage vs Panta_basic at 60.34% / 50.64%. Results reported across all 130 classes.",
       "supported": "moderate"
     },
     {
       "claim": "Hybrid program analysis (path selection) adds approximately 10% coverage improvement over iterative framework alone",
       "evidence": "Table 3 shows full Panta at 70.18% line / 60.83% branch vs Panta_basic at 60.34% / 50.64%, approximately 10pp improvement from adding static+dynamic path selection.",
       "supported": "moderate"
     },
     {
       "claim": "Claude 3.5 Haiku is the best-performing model across all metrics",
       "evidence": "Table 4 shows Claude achieves 73.20% line coverage, 66.52% branch coverage, 68.16% pass rate, 47.92% mutation score, and 62 HCC, leading all four models on every metric.",
       "supported": "moderate"
     },
     {
       "claim": "Panta maintains consistent coverage across complexity levels while other variants show significant drops for highly complex code",
       "evidence": "Table 3 complexity breakdown shows Panta with only 1pp drop for CYC_max > 20 (70.37% vs 69.42% line coverage) while baseline drops 6.6pp (34.01% vs 27.38%) and Panta_basic drops 6.1pp (61.56% vs 55.49%).",
       "supported": "moderate"
     },
     {
       "claim": "LLM-generated tests have weak fault detection capability, with mutation scores below 50% across all models",
       "evidence": "Table 4 shows average mutation scores of 43.79% (Llama), 36.60% (Mistral), 34.86% (GPT), and 47.92% (Claude). Section 5 discusses this limitation explicitly.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Re-implemented baseline",
       "detail": "The main baseline (SymPrompt) was re-implemented by the authors because neither SymPrompt nor HITS has publicly available code. This introduces self-comparison bias (Lucic et al. 2018 showed authors' re-implementations of baselines systematically underperform). The 26pp coverage gap is large enough that even a suboptimal re-implementation might not fully explain it, but the bias is unacknowledged."
     },
     {
       "flag": "Single run with no variance",
       "detail": "Despite using temperature=0.2 (not 0), all results are from a single run per class. The paper argues 'the generation process is largely deterministic' but provides no evidence for this claim. Minor fluctuations could compound across 130 classes."
     },
     {
       "flag": "No significance tests on main claims",
       "detail": "The central claim of 26%/23% improvement over SymPrompt is reported as raw percentage point differences without any statistical test. A paired t-test is used only in the ablation to show two variants are NOT different. The main claim of superiority lacks statistical support."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "Panta uses multiple LLM iterations per class (up to maxCYC iterations, with test repair loops), while SymPrompt makes single-pass LLM calls per method. The compute difference is substantial and undiscussed. A compute-matched comparison would be more informative."
     }
   ],
   "cited_papers": [
     {
       "title": "Automated unit test improvement using large language models at meta",
       "authors": ["Nadia Alshahwan", "Jubin Chheda", "Anastasia Finogenova", "Beliz Gokkaya", "Mark Harman", "Inna Harper", "Alexandru Marginean", "Shubho Sengupta", "Eddy Wang"],
       "year": 2024,
       "relevance": "Industry deployment of LLM-based test generation at Meta, evidence of LLM test generation in production."
     },
     {
       "title": "Chatunitest: A framework for llm-based test generation",
       "authors": ["Yinghao Chen", "Zehao Hu", "Chen Zhi", "Junxiao Han", "Shuiguang Deng", "Jianwei Yin"],
       "year": 2024,
       "relevance": "LLM-based test generation framework with generation-validation-repair mechanism."
     },
     {
       "title": "Leveraging Large Language Models for Enhancing the Understandability of Generated Unit Tests",
       "authors": ["Amirhossein Deljouyi", "Roham Koohestani", "Maliheh Izadi", "Andy Zaidman"],
       "year": 2024,
       "relevance": "Combines search-based testing with LLMs to improve test understandability."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "relevance": "Empirical evaluation of LLM test generation in JavaScript, documents coverage limitations."
     },
     {
       "title": "CodaMosa: Escaping Coverage Plateaus in Test Generation with Pre-Trained Large Language Models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K. Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "Hybrid approach combining search-based testing with LLMs to overcome coverage plateaus."
     },
     {
       "title": "Code-Aware Prompting: A Study of Coverage-Guided Test Generation in Regression Setting using LLM",
       "authors": ["Gabriel Ryan", "Siddhartha Jain", "Mingyue Shang", "Shiqi Wang", "Xiaofei Ma", "Murali Krishna Ramanathan", "Baishakhi Ray"],
       "year": 2024,
       "doi": "10.1145/3643769",
       "relevance": "SymPrompt: main baseline for comparison. Uses symbolic execution path constraints to guide LLM test generation."
     },
     {
       "title": "Coverup: Coverage-guided llm-based test generation",
       "authors": ["Juan Altmayer Pizzorno", "Emery D Berger"],
       "year": 2024,
       "relevance": "Coverage-guided LLM test generation using coverage reports to indicate uncovered code."
     },
     {
       "title": "HITS: High-coverage LLM-based Unit Test Generation via Method Slicing",
       "authors": ["Zejun Wang", "Kaibo Liu", "Ge Li", "Zhi Jin"],
       "year": 2024,
       "relevance": "LLM-based test generation using method decomposition into slices."
     },
     {
       "title": "ASTER: Natural and Multi-language Unit Test Generation with LLMs",
       "authors": ["Rangeet Pan", "Myeongsoo Kim", "Rahul Krishna", "Raju Pavuluri", "Saurabh Sinha"],
       "year": 2025,
       "arxiv_id": "2409.03093",
       "relevance": "Multi-language LLM test generation pipeline for complex software systems."
     },
     {
       "title": "Using large language models to generate junit tests: An empirical study",
       "authors": ["Mohammed Latif Siddiq", "Joanna Cecilia Da Silva Santos", "Ridwanul Hasan Tanvir", "Noshin Ulfat", "Fahmid Al Rifat", "Vinícius Carvalho Lopes"],
       "year": 2024,
       "relevance": "Empirical study of LLM-generated JUnit tests documenting coverage challenges."
     },
     {
       "title": "Testeval: Benchmarking large language models for test case generation",
       "authors": ["Wenhan Wang", "Chenyuan Yang", "Zhijie Wang", "Yuheng Huang", "Zhaoyang Chu", "Da Song", "Lingming Zhang", "An Ran Chen", "Lei Ma"],
       "year": 2024,
       "relevance": "Benchmark for evaluating LLM test generation capabilities across programming languages."
     },
     {
       "title": "Large Language Models are Few-Shot Testers: Exploring LLM-Based General Bug Reproduction",
       "authors": ["Sungmin Kang", "Juyeon Yoon", "Shin Yoo"],
       "year": 2023,
       "doi": "10.1109/ICSE48619.2023.00194",
       "relevance": "LLM-based bug reproduction from bug reports, related capability for test generation."
     }
   ]
 }