ai-research-survey

scan.json (35705B)

---

 {
   "paper": {
     "title": "Tracking the Moving Target: A Framework for Continuous Evaluation of LLM Test Generation in Industry",
     "authors": [
       "Maider Azanza",
       "Beatriz Pérez Lamancha",
       "Eneko Pizarro"
     ],
     "year": 2025,
     "venue": "International Conference on Evaluation & Assessment in Software Engineering (EASE 2025)",
     "arxiv_id": "2504.18985",
     "doi": "10.1145/3756681.3756946"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided at Zenodo (https://zenodo.org/records/15274212), referenced in Section 5.2: 'The full reference to the methods can be found in the replication package.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The replication package at Zenodo contains the test methods and related materials. The paper also states the final optimized prompt is available in the replication package (Section 5.4)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 specifies the tooling infrastructure: JUnit 5, SonarQube/SonarCloud, JaCoCo, Maven for dependency management. 'All dependency management was handled through Maven, ensuring reproducible builds and consistent tool versions across evaluations.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the methodology (Section 5) but does not provide step-by-step reproduction instructions. The replication package is referenced but the paper itself contains no README-style commands or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 5 reports only point estimates for all metrics (e.g., '39.14%' line coverage, '91.76%' total weight assessment). No confidence intervals, error bars, or ± notation appear anywhere in the results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 6 model configurations across 11 metrics (Table 5) and draws comparative conclusions ('dramatic improvements,' 'most immediate practical outcome') without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides baseline context for understanding improvement magnitude — e.g., compilation errors from 31 to 0, line coverage from 39.14% to 98.00%, total weighted assessment from 32.44% to 91.76%. The reader can compute effect sizes from the provided numbers."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 justifies the sample: 'Due to the significant time requirements of manual expert evaluation needed for measuring subjective metrics, we limited our selection to seven functions. These were specifically chosen to cover a diverse range of typical industrial programming patterns and testing challenges.'"
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, IQR, or spread measures are reported. Table 5 shows single aggregate values per model/metric with no indication of variability across the 7 test functions or across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: the initial GPT-4 evaluation (March 2024) serves as a temporal baseline, expert-written tests serve as a quality baseline (Expert-generated Test Coverage metric), and 6 model configurations are compared against each other in Table 5."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The December 2024 evaluation includes contemporary models: GPT-o, o1-Preview, o1-Mini, and Claude 3.5 Sonnet. These were among the most capable models available at evaluation time."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents a measurement framework (an evaluation instrument), not a multi-component system that can be ablated. The LLMs being evaluated are third-party black boxes whose components cannot be removed."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The framework uses 11 metrics across 3 categories: code quality metrics (compilation errors, static analysis issues, setup/teardown usage), white box metrics (line/branch/decision coverage, test isolation), and black box metrics (equivalence partitioning, boundary value analysis, parameterization, expert coverage)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Black box metrics (Section 4.3) require expert judgment: 'Unlike code quality and white box metrics that can be automatically computed, these metrics require expert judgment to evaluate test design decisions and methodology.' Expert QA engineers evaluated equivalence partitioning, boundary value analysis, parameterization, and expert-generated test coverage."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The same 7 functions used for iterative prompt refinement (Section 5.3-5.4) are used for final evaluation. There is no separation between development and test sets — prompt engineering was tuned on the same examples used to report results."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 5 provides per-metric breakdowns across all 11 metrics and 6 model configurations, with separate sections for code quality, white box, and black box metrics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses early failures: 'In March 2024, GitHub Copilot generated tests that largely failed to compile and required significant developer intervention to be usable.' Section 5.4 details specific issues like compilation errors, static analysis warnings, and low coverage that prompted prompt refinement."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The March 2024 results (32.44% total score, 31 compilation errors) are effectively negative results. The o1-Mini model performed poorly (63.81% total, 28.57% on coverage metrics) compared to peers, and this is reported without being hidden."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims the framework 'integrates with industry-standard tools like SonarQube' (supported by Section 4-5), 'provides metrics that evaluate both technical adequacy and practical considerations' (supported by Tables 1-3), and 'results highlight the rapid evolution of LLM capabilities' (supported by Table 5's longitudinal comparison)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 claims 'proper prompt engineering significantly impacts performance,' and the March→May comparison implies prompt refinement caused improvement. However, Section 8 acknowledges: 'The observed improvements in LLM performance over time could be influenced by factors other than actual evolution in LLM capabilities. In particular, our iterative refinement of prompting strategies might confound the results.' The confound is acknowledged but not resolved."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'in Industry' (general) and the abstract offers 'practical guidance for companies seeking to integrate these technologies' (plural, general). However, results are from 7 Java functions at a single company (LKS Next). While Section 8 acknowledges the single-organization limitation, the title and abstract frame the results more broadly than the evidence supports."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 (Internal Validity) discusses a specific alternative explanation: 'The observed improvements in LLM performance over time could be influenced by factors other than actual evolution in LLM capabilities. In particular, our iterative refinement of prompting strategies might confound the results.' They describe mitigation measures including consistent base prompts and documented modifications."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges the gap between metrics and quality: 'Coverage vs. effectiveness gap: Achieving high coverage does not necessarily guarantee meaningful tests' (Section 2.3). The dual objective/subjective metric design (Section 4) is itself a response to the proxy-outcome gap, recognizing that automated metrics alone don't capture test quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referenced by marketing names only: 'GPT-4,' 'GPT-o,' 'o1-Preview,' 'o1-Mini,' 'Claude 3.5 Sonnet.' No specific version strings (e.g., 'gpt-4-0613'), snapshot dates, or API versions are provided. Model behavior changes across versions, making these designations insufficient for reproduction."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 includes substantial quoted prompt fragments (e.g., 'When creating the test class, do not generate any line of code that could create any Issue, nor Bug nor Code Smell'). The paper states 'The final optimized prompt (available in our replication package)' at the Zenodo URL, providing the full prompt text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters are reported — no temperature, top-p, max tokens, or other API settings. The paper uses commercial LLM tools (GitHub Copilot, ChatGPT) without stating their sampling parameters, which significantly affect output."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates GitHub Copilot and ChatGPT as third-party black-box tools. Per the schema, papers evaluating third-party tools as black boxes are NA — the authors cannot be expected to describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 documents the test case selection process in detail, explaining criteria (avoiding data leakage, representing common industrial patterns, compatibility with unit and integration testing) and listing each of the 7 functions with specific rationale for inclusion."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' provides substantive discussion organized across four dimensions: construct validity, internal validity, external validity, and reliability, each with specific threats and mitigation strategies."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses threats specific to this study: 'our iterative refinement of prompting strategies might confound the results' (internal validity), 'primarily based on experiences from a single organization and specific types of software projects' (external validity), 'expert assessment component inherently involves subjective judgment' (reliability)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states generic limitations ('may not generalize to organizations with different development practices, project types, or quality requirements') but does not state specific exclusions — e.g., it doesn't say 'we did not test languages other than Java,' 'we did not evaluate system-level testing,' or 'our results do not apply to non-enterprise codebases.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is available at Zenodo (https://zenodo.org/records/15274212), referenced in Section 5.2. The paper states the methods and prompt are available there for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 describes the full evaluation workflow: environment setup (5.1), test case selection with specific criteria (5.2), iterative evaluation process (5.3), prompt engineering methodology (5.4), and tooling infrastructure (5.5). The study period (March 2024 to December 2024) is specified."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Expert evaluators are identified only as 'expert Quality Assurance engineers from LKS Next' and 'a QA team leader.' The paper does not describe how these experts were selected, their qualifications, experience level, or whether the selection could introduce bias in subjective metric evaluation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 shows the BPMN diagram of the iterative measurement process, and Section 5 documents each stage: environment preparation → test case selection and ground truth → iterative evaluation (generate → analyze → compare → refine/document). The pipeline from generation to final scoring is traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists specific funding: MCIN/AEI/10.13039/501100011033 contract, EU NextGeneration EU/PRTR under PID2021-125438OB-I00, University of the Basque Country 'University-Enterprise-Society' program (US24/10), and grant GIU21/037."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Maider Azanza and Eneko Pizarro from University of the Basque Country (UPV/EHU), Beatriz Pérez Lamancha from LKS Next. The dual academic-industry affiliation is transparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from government/EU grants (MCIN/AEI, EU NextGeneration) and university programs. These funders have no financial stake in whether GitHub Copilot or other LLMs perform well at test generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or conflicts-of-interest statement is provided. One author is employed by LKS Next, which is the company evaluating tools for potential adoption — this creates a potential interest in finding the tools viable, but no declaration addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models (GPT-4, GPT-o, o1-Preview, o1-Mini, Claude 3.5 Sonnet). The paper discusses contamination risk in general but does not specify when each model's training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 explicitly addresses this: 'Our initial explorations confirmed previous findings that LLMs trained on public repositories often reproduce existing tests rather than generating novel ones, especially when encountering familiar code. To ensure our evaluation measured true generative ability, we deliberately selected functions and classes from projects that did not have pre-existing, publicly accessible test suites.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The contamination risk is mitigated by design: the evaluation uses private industrial code from LKS Next that would not be in LLM training data. Section 5.2 describes selecting 'functions and classes from projects that did not have pre-existing, publicly accessible test suites.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The expert evaluators are part of the research methodology (assessors), not study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. The paper evaluates LLM-generated tests using automated tools and expert assessment, not human subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. Expert evaluators are part of the research team, not study participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied. This is an evaluation of LLM tools, not an experimental study with human subjects."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were studied."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 mentions 'higher computational costs and latency' for newer models and 'per-seat licensing' for GitHub Copilot, but no actual cost figures are reported — no API costs, tokens consumed, or cost per test suite generated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper does not report GPU hours, API spend, evaluation wall-clock time, or hardware used for the evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "LLM outputs are non-deterministic, but the paper reports no analysis of variability across runs. Table 5 shows single values per model/metric with no indication of run-to-run variance."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs per model configuration is not stated. Section 8 mentions 'validate significant changes in performance through multiple test runs' as a mitigation strategy but provides no specific numbers."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The prompt engineering process is described as iterative (Section 5.4) but no search budget is reported — the number of prompt iterations tried, total configurations explored, or compute spent on prompt refinement is not stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The iterative prompt refinement process (Section 5.3-5.4) describes refining until 'quality thresholds are met,' but doesn't report all configurations tried or how the final prompt was selected over alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares 6 model configurations across 11 metrics (66 implicit comparisons) without any statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed the evaluation framework and then used it to evaluate LLMs. They do not acknowledge the inherent bias of evaluating tools using their own framework — the metric weights and selection naturally favor the dimensions they chose to measure."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 acknowledges 'newer models like Claude 3.5 Sonnet and o1-Preview show impressive performance' but 'come with higher computational costs and latency.' However, no quantitative performance-per-compute comparison is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2-2.3 discusses construct validity: 'Coverage vs. effectiveness gap: Achieving high coverage does not necessarily guarantee meaningful tests.' The framework explicitly addresses this by including both automated metrics and expert assessment, questioning whether coverage alone measures test quality."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper is unclear about whether all models were evaluated through the same interface (GitHub Copilot) or some via direct ChatGPT access. Table 5 headers ('ChatGPT-4' vs model names) suggest mixed interfaces. The scaffold/interface confound is not discussed."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Temporal leakage is mitigated by design. Section 5.2: 'we deliberately selected functions and classes from projects that did not have pre-existing, publicly accessible test suites,' meaning the test targets would not appear in any model's training data regardless of cutoff date."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt provides extensive context: full source code, all interconnected classes, JavaDoc, Maven dependencies, framework documentation links, and Mockito usage examples (Section 5.4). No discussion of whether this rich context represents realistic usage or gives the LLM information beyond what a typical developer would provide."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The 7 test functions appear to come from different projects and domains (rental management, user management, prime checker, etc.), but the paper does not explicitly discuss independence between test cases or whether shared company conventions could create dependencies."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes a concrete prevention method: 'Our initial explorations confirmed previous findings that LLMs trained on public repositories often reproduce existing tests rather than generating novel ones.' They then used private industrial code without public test suites as a mitigation strategy."
       }
     },
     "survey_methodology": {
       "prisma_or_structured_protocol": {
         "applies": false,
         "answer": false,
         "justification": "This is not a survey or systematic review paper. It is an industrial case study evaluating LLM test generators."
       },
       "quality_assessment_of_sources": {
         "applies": false,
         "answer": false,
         "justification": "This is not a survey or systematic review paper."
       },
       "publication_bias_discussed": {
         "applies": false,
         "answer": false,
         "justification": "This is not a survey or systematic review paper."
       }
     }
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "claims": [
     {
       "claim": "LLM test generation capabilities improved dramatically from March 2024 to December 2024, with weighted assessment rising from 32.44% to over 91%.",
       "evidence": "Table 5 shows progression: ChatGPT-4 (March 2024) scored 32.44%, iterative GPT-4 (May 2024) 67.96%, o1-Preview (December 2024) 91.76%, Claude 3.5 Sonnet 90.72%.",
       "supported": "moderate"
     },
     {
       "claim": "Iterative prompt engineering significantly improves test generation quality within the same model.",
       "evidence": "Table 5: GPT-4 improved from 32.44% (first time) to 67.96% (iterative) using the same model with refined prompts. Section 5.4 details the prompt refinement process.",
       "supported": "moderate"
     },
     {
       "claim": "o1-Preview and Claude 3.5 Sonnet achieve near-perfect compilation with 0 errors and line coverage above 95%.",
       "evidence": "Table 5: o1-Preview has 0 compilation errors and 98.00% line coverage; Claude 3.5 Sonnet has 0 compilation errors and 95.71% line coverage.",
       "supported": "weak"
     },
     {
       "claim": "Test parameterization improved from 12.70% to 88-91% across the study period.",
       "evidence": "Table 5 shows parameterization metric going from 12.70% (March 2024 GPT-4) to 91.84% (o1-Preview) and 88.89% (Claude 3.5 Sonnet).",
       "supported": "moderate"
     },
     {
       "claim": "English prompts produce significantly better results compared to Spanish prompts.",
       "evidence": "Section 5.4 states 'Our analysis revealed significantly better results with English prompts compared to Spanish' but provides no quantitative comparison data.",
       "supported": "unsupported"
     },
     {
       "claim": "Expert oversight remains essential despite high automated scores from newer models.",
       "evidence": "Section 7 discusses this qualitatively, noting black box metrics show LLMs 'don't yet match human expertise in test design decisions.' The gap between automated and expert test coverage metrics provides some evidence.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "This paper presents a measurement framework for continuously evaluating LLM-based test generators in industrial settings, validated through a longitudinal study at LKS Next using 7 Java functions. Results show dramatic improvement from March to December 2024: GPT-4 initially scored 32.44% on a weighted assessment combining 11 code quality, coverage, and testing design metrics, while newer models (o1-Preview, Claude 3.5 Sonnet) achieved over 90%. The study demonstrates the importance of prompt engineering (same model improved from 32% to 68% through prompt refinement) and highlights that expert oversight remains necessary despite high automated metric scores.",
   "red_flags": [
     {
       "flag": "Tiny sample size",
       "detail": "Only 7 Java functions were evaluated. This is an extremely small sample for drawing conclusions about LLM test generation capability in industry. The paper acknowledges the limitation (resource constraints of manual expert evaluation) but the sample is too small for the generality of claims made."
     },
     {
       "flag": "Single organization study with broad claims",
       "detail": "All evaluation was conducted at LKS Next, a single mid-sized consultancy. The title frames this as 'in Industry' generally, and the abstract offers 'practical guidance for companies,' but the evidence base is one company with one technology stack (Java, Maven, SonarQube)."
     },
     {
       "flag": "Prompt engineering confounds temporal comparison",
       "detail": "The March→May GPT-4 improvement (32.44%→67.96%) conflates model improvement with prompt refinement. The December evaluation uses the refined prompt on newer models, making it impossible to isolate how much improvement is due to better models vs. better prompts. Section 8 acknowledges this but doesn't resolve it."
     },
     {
       "flag": "No statistical analysis on any comparison",
       "detail": "All results are point estimates from a single evaluation per model configuration across 7 functions. No error bars, confidence intervals, significance tests, or variance measures are reported despite making comparative claims about model performance."
     },
     {
       "flag": "Company evaluating tools for own adoption",
       "detail": "One author (Beatriz Pérez Lamancha) is from LKS Next, the company evaluating LLM tools for potential adoption. This creates a subtle incentive bias — the company benefits from finding the tools viable. The paper reports increasingly positive results culminating in the company 'reconsidering' adoption."
     },
     {
       "flag": "No held-out evaluation",
       "detail": "The same 7 functions were used both for iterative prompt refinement and for final evaluation. There is no held-out test set, meaning the reported scores may be inflated by prompts tuned specifically for these examples."
     }
   ],
   "cited_papers": [
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["J. Wang", "Y. Huang", "C. Chen", "Z. Liu", "S. Wang", "Q. Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of 102 papers on LLM-based software testing, identifying key challenges including coverage, test oracle problem, and real-world application."
     },
     {
       "title": "Using large language models to generate junit tests: An empirical study",
       "authors": ["M. Siddiq", "J. Da Silva", "R. Tanvir", "N. Ulfat", "F. Al Rifat", "V. Carvalho"],
       "year": 2024,
       "relevance": "Empirical evaluation of LLM unit test generation (Codex, GPT-3.5-Turbo, StarCoder) highlighting challenges with coverage, compilability, and reliability."
     },
     {
       "title": "ChatGPT vs SBST: A comparative assessment of unit test suite generation",
       "authors": ["Y. Tang", "Z. Liu", "Z. Zhou", "X. Luo"],
       "year": 2024,
       "doi": "10.1109/TSE.2024.3382365",
       "relevance": "Compares ChatGPT-generated tests against search-based software testing tools, finding SBST achieves higher coverage but LLM tests show better readability and assertion quality."
     },
     {
       "title": "ChatUniTest: A framework for LLM-based test generation",
       "authors": ["Y. Chen", "Z. Hu", "C. Zhi", "J. Han", "S. Deng", "J. Yin"],
       "year": 2024,
       "relevance": "Framework addressing LLM test generation limitations through adaptive focal context and generation-validation-repair mechanism."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["M. Schäfer", "S. Nadi", "A. Eghbali", "F. Tip"],
       "year": 2024,
       "doi": "10.1109/TSE.2023.3334955",
       "relevance": "TESTPILOT system for JavaScript test generation using LLMs, demonstrating importance of prompt engineering and iterative refinement for test quality."
     },
     {
       "title": "AutoTestGPT: A system for the automated generation of software test cases based on ChatGPT",
       "authors": ["H. Liu", "L. Liu", "C. Yue", "Y. Wang", "B. Deng"],
       "year": 2024,
       "relevance": "System using ChatGPT for automated test case generation, achieving 70% reduction in framework generation time vs. manual approaches."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["X. Hou", "Y. Zhao", "Y. Liu", "Z. Yang", "K. Wang", "L. Li", "X. Luo", "D. Lo", "J. Grundy", "H. Wang"],
       "year": 2024,
       "relevance": "Comprehensive systematic literature review of LLMs applied to software engineering tasks."
     },
     {
       "title": "The current challenges of software engineering in the era of large language models",
       "authors": ["C. Gao", "X. Hu", "S. Gao", "X. Xia", "Z. Jin"],
       "year": 2025,
       "relevance": "Examines current challenges that LLMs pose for software engineering practices and research."
     },
     {
       "title": "Using AI-based coding assistants in practice: State of affairs, perceptions, and ways forward",
       "authors": ["A. Sergeyuk", "Y. Golubev", "T. Bryksin", "I. Ahmed"],
       "year": 2025,
       "relevance": "Survey on practical use of AI coding assistants, finding test writing is one of the tasks developers most want to delegate to AI."
     },
     {
       "title": "On inter-dataset code duplication and data leakage in large language models",
       "authors": ["J. A. H. López", "B. Chen", "M. Saad", "T. Sharma", "D. Varró"],
       "year": 2025,
       "relevance": "Addresses data leakage and code duplication concerns when evaluating LLMs, directly relevant to benchmark contamination assessment."
     },
     {
       "title": "ConDefects: A complementary dataset to address the data leakage concern for LLM-based fault localization and program repair",
       "authors": ["Y. Wu", "Z. Li", "J. M. Zhang", "Y. Liu"],
       "year": 2024,
       "relevance": "Dataset designed to mitigate data leakage in LLM evaluation for fault localization and program repair."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "The measurement framework is directly applicable for companies evaluating LLM test generation tools in their development pipelines."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that LLM test generation improved from 32% to 91% in 9 months is noteworthy but follows expected improvement trends rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns are raised; the paper focuses on tool evaluation methodology."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper presents a constructive framework without challenging any specific claims or organizations."
     },
     "demo_ability": {
       "score": 1,
       "justification": "A replication package is available at Zenodo, but the framework is a methodology/process rather than an installable tool."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "The study evaluates GitHub Copilot (widely known), GPT-4, and Claude 3.5 Sonnet — recognizable products — though the authors and company (LKS Next) are not widely known."
     }
   }
 }