ai-research-survey

scan.json (30488B)

---

 {
   "paper": {
     "title": "HITS: High-coverage LLM-based Unit Test Generation via Method Slicing",
     "authors": [
       "Zejun Wang",
       "Kaibo Liu",
       "Ge Li",
       "Zhi Jin"
     ],
     "year": 2024,
     "venue": "International Conference on Automated Software Engineering",
     "arxiv_id": "2408.11324",
     "doi": "10.1145/3691620.3695501"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "HITS decomposes complex Java methods into slices and generates unit tests per slice using an LLM, achieving 55.09% average line coverage and 48.12% branch coverage on complex methods—outperforming ChatUniTest (32.48%/27.07%), ChatTester (20.71%/18.20%), SymPrompt (26.32%/25.10%), and Evosuite (39.10%/38.46%). Notably, existing LLM-based baselines perform worse than the traditional SBST tool Evosuite on complex methods, reversing the advantage seen on average-complexity methods. The ablation study confirms that the slicing decomposition is the primary contributor to improvements.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides an anonymous artifact link: https://anonymous.4open.science/r/SlicePromptTest4J-6CF1/. While an anonymous review link, it constitutes a provided URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset comprises complex methods extracted from 10 open-source projects (Table 2), all publicly available. The artifact link likely includes the dataset specification. The base projects (Commons-CLI, Gson, etc.) are all public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions gpt-turbo-3.5-0125, Java, JUnit 5, Jacoco, and Evosuite but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The anonymous artifact link may contain a README, but the paper itself does not describe how to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 4-7 are point estimates with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims HITS 'significantly outperforms' baselines, but no statistical significance tests (p-values, t-tests, etc.) are reported anywhere. Comparisons are based solely on comparing raw numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute coverage scores with baseline comparisons (e.g., HITS 55.09% vs ChatUniTest 32.48% line coverage) and describes improvements as 'a percentage ranging from 10 to 20,' providing enough context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The dataset contains 120 complex methods from 10 projects. No power analysis or sample size justification is provided. Budget constraints are mentioned as a limitation but not as a justification."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy generation is used (deterministic single-run), so no variance across runs is reported. No standard deviation, IQR, or spread measures appear in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are included: ChatUniTest, ChatTester, SymPrompt (LLM-based), and Evosuite (SBST). Results are compared across all in Tables 4-6."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ChatUniTest (2023), ChatTester (2023), and SymPrompt (2024) are recent LLM-based methods. Evosuite is an established and widely-used SBST baseline."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 presents an ablation study with two configurations: 'w/o slicing' and 'w/o slicing & PE' (equivalent to ChatUniTest + PP), measuring the contribution of slicing, prompt engineering, and post-processing."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: line coverage (Table 4), branch coverage (Table 5), and pass rate (Table 6)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated test quality is performed. All evaluation is automated via Jacoco coverage measurement and execution correctness checking."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "There is no explicit separation between prompt development data and evaluation data. The prompts and examples were manually crafted, and it is unclear whether any of the 10 evaluation projects were used during prompt development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4-7 provide per-project breakdowns for all 10 projects, and Table 2 splits projects into 'Learned' (in training data) and not learned."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 provides a case study showing where ChatUniTest fails and how HITS addresses it. Section 5.3 analyzes the distribution of non-executable tests (compilation vs runtime errors)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that Evosuite outperforms all LLM-based methods on several specific projects (e.g., COD, DAT). The ablation study shows the slicing workflow hurts branch coverage on a few projects (COL, JDO). Section 4.4 analyzes why."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims HITS 'significantly outperforms current test case generation methods with LLMs and the typical SBST method Evosuite regarding both line and branch coverage scores.' Tables 4 and 5 show HITS averaging 55.09%/48.12% vs next best Evosuite at 39.10%/38.46%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims slicing 'improves' coverage (causal language). The ablation study (Table 7) removes slicing while keeping other components, measuring the isolated effect. This controlled single-variable manipulation is adequate for the causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('LLM-based Unit Test Generation') implies broad applicability, but results are limited to 10 Java projects tested with a single LLM (gpt-3.5-turbo). The method relies on Java-specific tooling (JUnit 5, Mockito, Jacoco). The threats section acknowledges limited LLM generalization but the title and framing remain broad."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 discusses threats (limited dataset, limited LLMs) but these are methodological limitations, not alternative explanations for the results. The paper does not discuss confounds like whether improvements come from longer prompts providing more context, or whether the chain-of-thought approach rather than slicing per se drives the gains."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures line and branch coverage as the primary outcomes and implicitly equates these with test quality. No discussion of the gap between coverage and actual fault detection ability, mutation score, or other measures of test effectiveness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-turbo-3.5-0125' as the model, which includes the specific version/snapshot identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 2 shows prompt templates with structural placeholders ({{ focal method }}, {{ dependencies }}) and abbreviated examples ('... (the example)'). The actual filled prompts and hand-crafted examples referenced in the prompts are not fully provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'greedy generation first' and 'raise the top-p slowly and gradually' on format violations, but does not specify the actual top-p values used. Max fixing rounds (10) are stated, but other API parameters (max tokens, etc.) are not reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-step pipeline is described in detail: static analysis for context retrieval, chain-of-thought method decomposition into slices, per-slice test generation with scenario analysis, self-debug for broken tests, and post-processing (format extraction, test isolation, rule-based fixing). Figure 2 provides an overview diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the dataset construction: projects selected by domain overlap with prior work, 1-30 complex methods, high GitHub stars. Complex methods defined as cyclomatic complexity > 10. Table 2 shows per-project statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 'Threats to Validity' provides a dedicated discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 identifies specific threats: limited to projects with ≤30 complex methods due to budget, gpt-3.5-turbo chosen for cost/capability tradeoff, and notes that 'other LLMs are either incapable of understanding the instructions of HITS to generate or are much more expensive.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. The threats section mentions limitations (limited dataset, limited LLMs) but doesn't draw explicit scope boundaries like 'these results do not generalize to non-Java languages' or 'we do not claim fault detection effectiveness.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While an anonymous artifact link is provided, the paper does not explicitly state that raw experimental data (individual test outputs, coverage reports per method, LLM responses) are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes three project selection standards: (1) domain used in Evosuite/ChatUniTest, (2) contains 1-30 complex methods, (3) prioritize high-star GitHub projects. Complex methods defined via cyclomatic complexity > 10."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "The sample selection process is described: projects crawled from the Internet following three explicit standards (domain overlap, method count range, GitHub stars). Table 2 provides per-project metadata including domain and version."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the selection criteria but does not document the full pipeline from initial candidate pool to final dataset. There's no accounting of how many candidate projects were considered, how many were filtered at each stage, or the total methods before complexity filtering."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. The authors mention 'limited budget of the experiment fund' in Section 5.4, implying funding exists but the source is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as affiliated with Key Lab of HCST (PKU), MOE; SCS, Beijing, China. No conflict with evaluated products (they use OpenAI API but don't work at OpenAI)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed (despite mentioning an 'experiment fund'), independence of the funder cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper references 'the cutoff date of the training data of gpt-turbo-3.5' and splits projects by learned/not-learned, but never states the actual cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 explicitly marks which projects were 'Learned' (created before training cutoff) and which were not. The paper discusses how projects in the training set may allow the LLM to 'recall' tests from memory (Section 4.4)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper designs the experiment around contamination: 6 projects in training data, 4 after cutoff. Results are reported separately. Section 4.4 discusses how 'Commons-collections is more broadly used' and the LLM may rely on training data recall rather than following instructions."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of automated test generation tools."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time are reported despite the method making many LLM calls per focal method (decomposition + per-slice generation + fixing rounds)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget is not quantified. The paper mentions 'limited budget of the experiment fund' but provides no figures for total API spend, tokens, or compute hours."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Greedy generation is used (deterministic), so only a single run is performed. No analysis of sensitivity to random seeds or non-deterministic settings."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'greedy generation first' implying a single run, but does not explicitly state the number of experimental runs. When top-p is raised for format violations, the number of retries is not specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. The prompts and examples were manually crafted, but no search budget or number of configurations tried is stated."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents a single configuration without explaining how it was selected or whether alternatives were tried. Prompt design choices appear final without justification for selection."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so correction for multiple comparisons is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implemented SymPrompt ('We implement SymPrompt since we have found no implementations') but do not acknowledge the bias of evaluating their own implementation of a baseline. ChatUniTest and ChatTester use open-source implementations."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 shows test case counts but not compute costs. HITS makes multiple LLM calls per method (decomposition + per-slice generation + fixing) while baselines use different strategies. No performance-vs-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses line and branch coverage as the sole evaluation criteria without discussing whether coverage actually measures test effectiveness, fault detection ability, or test quality. The proxy gap between coverage and real-world test value is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All LLM-based methods use the same model (gpt-turbo-3.5-0125), isolating the scaffolding/prompting approach as the experimental variable. The comparison explicitly tests different scaffolding strategies under a controlled model."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper splits projects into 6 'Learned' (before training cutoff) and 4 'Not Learned' (after cutoff) and reports results for both groups, directly addressing temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context. The test generation task provides source code as input (standard practice), but potential leakage through project-level context is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether methods within the same project share structural similarities that could inflate results, or whether training data for the LLM includes these specific projects' test suites."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splitting is used as a concrete prevention method: projects are categorized by whether they were created before or after the model's training data cutoff date."
       }
     }
   },
   "claims": [
     {
       "claim": "HITS outperforms all baselines on both line and branch coverage scores for complex methods.",
       "evidence": "Table 4: HITS 55.09% avg line coverage vs ChatUniTest 32.48%, ChatTester 20.71%, SymPrompt 26.32%, Evosuite 39.10%. Table 5: HITS 48.12% avg branch coverage vs ChatUniTest 27.07%, ChatTester 18.20%, SymPrompt 25.10%, Evosuite 38.46%.",
       "supported": "moderate"
     },
     {
       "claim": "Existing LLM-based test generation methods are inferior to the SBST method Evosuite when testing complex methods.",
       "evidence": "Tables 4-5: Evosuite (39.10%/38.46%) outperforms ChatUniTest (32.48%/27.07%), ChatTester (20.71%/18.20%), and SymPrompt (26.32%/25.10%) on average, contrary to prior claims of LLM superiority on average-complexity methods.",
       "supported": "moderate"
     },
     {
       "claim": "The slicing decomposition is the primary contributor to HITS's coverage improvements.",
       "evidence": "Table 7 ablation: HITS 55.09%/48.12% vs w/o slicing 50.61%/42.97% vs w/o slicing & PE (ChatUniTest+PP) 48.56%/42.38%. Slicing alone adds ~4.5pp line and ~5pp branch coverage on average.",
       "supported": "moderate"
     },
     {
       "claim": "HITS has a higher pass rate (proportion of executable tests) than all LLM-based baselines.",
       "evidence": "Table 6: HITS 69.11% average pass rate vs ChatUniTest 41.82%, SymPrompt 21.00%, ChatTester 16.10%.",
       "supported": "moderate"
     },
     {
       "claim": "Generating tests slice by slice reduces the input condition combinations from multiplicative to additive, simplifying the LLM's analysis task.",
       "evidence": "Section 3.2 provides the theoretical argument: for n slices with ni conditions each, slice-by-slice inference requires Σni combinations vs Πni for whole-method inference. The case study (Section 5.1, Figure 3) illustrates this concretely.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The abstract claims HITS 'significantly outperforms' baselines, but no statistical tests are performed. All comparisons are raw number comparisons across 10 projects with no p-values, t-tests, or bootstrap tests."
     },
     {
       "flag": "Small dataset with selection bias",
       "detail": "Only 10 projects with 120 complex methods, selected to have 1-30 complex methods (excluding larger projects due to budget). This selection criterion may bias toward smaller, simpler projects."
     },
     {
       "flag": "Single-LLM evaluation",
       "detail": "Only gpt-3.5-turbo is evaluated. The paper acknowledges 'other LLMs are either incapable of understanding the instructions of HITS to generate or are much more expensive,' suggesting the method may not generalize."
     },
     {
       "flag": "Self-implemented baseline (SymPrompt)",
       "detail": "The authors re-implemented SymPrompt since no public implementation exists. This introduces potential bias, as authors' implementations of baselines systematically underperform (Lucic et al., 2018)."
     },
     {
       "flag": "Single deterministic run with no variance reporting",
       "detail": "Greedy generation produces a single deterministic result per configuration. No multiple runs, no variance, no error bars. The stability of results is unknown."
     },
     {
       "flag": "No cost reporting despite multi-call pipeline",
       "detail": "HITS makes multiple LLM calls per method (decomposition + per-slice generation + up to 10 fixing rounds per broken test) but reports no API costs, making practical cost comparison with baselines impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "ChatUniTest: a ChatGPT-based automated unit test generation tool",
       "authors": ["Zhuokui Xie", "Yinghao Chen", "Chen Zhi", "Shuiguang Deng", "Jianwei Yin"],
       "year": 2023,
       "arxiv_id": "2305.04764",
       "relevance": "Primary baseline and inspiration for HITS; practical LLM-based Java unit test generation tool."
     },
     {
       "title": "No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation",
       "authors": ["Zhiqiang Yuan", "Yiling Lou", "Mingwei Liu", "Shiji Ding", "Kaixin Wang", "Yixuan Chen", "Xin Peng"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "ChatTester baseline; evaluates ChatGPT for unit test generation showing LLMs outperform SBST on average methods."
     },
     {
       "title": "Code-Aware Prompting: A study of Coverage Guided Test Generation in Regression Setting using LLM",
       "authors": ["Gabriel Ryan", "Siddhartha Jain", "Mingyue Shang", "Shiqi Wang", "Xiaofei Ma", "Murali Krishna Ramanathan", "Baishakhi Ray"],
       "year": 2024,
       "arxiv_id": "2402.00097",
       "relevance": "SymPrompt baseline; uses execution paths as scaffolds for LLM-based test generation."
     },
     {
       "title": "Evolutionary Generation of Whole Test Suites",
       "authors": ["Gordon Fraser", "Andrea Arcuri"],
       "year": 2011,
       "doi": "10.1109/QSIC.2011.19",
       "relevance": "Evosuite SBST baseline; the standard search-based test generation tool compared against LLM methods."
     },
     {
       "title": "Codamosa: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "Hybrid SBST+LLM approach that calls LLM when traditional tools plateau, relevant to LLM-assisted test generation."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "relevance": "Empirical study of LLMs for unit test generation, establishing baseline expectations for LLM test quality."
     },
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["Junjie Wang", "Yuchao Huang", "Chunyang Chen", "Zhe Liu", "Song Wang", "Qing Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based software testing methods and landscape."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2024,
       "relevance": "Self-Debug technique used by HITS to fix broken generated tests via LLM self-repair."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Self-Refine technique used by HITS for prompt optimization (Section 5.2)."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper establishing foundational evaluation of LLMs for code generation."
     },
     {
       "title": "Unit test case generation with transformers and focal context",
       "authors": ["Michele Tufano", "Dawn Drain", "Alexey Svyatkovskiy", "Shao Kun Deng", "Neel Sundaresan"],
       "year": 2020,
       "relevance": "ATHENATEST — early deep learning approach to unit test generation that outperformed Evosuite on some tasks."
     },
     {
       "title": "ChatGPT vs SBST: A comparative assessment of unit test suite generation",
       "authors": ["Yutian Tang", "Zhijie Liu", "Zhichao Zhou", "Xiapu Luo"],
       "year": 2024,
       "relevance": "Direct comparison of ChatGPT vs search-based testing, relevant to the LLM-vs-SBST comparison in this paper."
     },
     {
       "title": "Feedback-Directed Random Test Generation",
       "authors": ["Carlos Pacheco", "Shuvendu K. Lahiri", "Michael D. Ernst", "Thomas Ball"],
       "year": 2007,
       "doi": "10.1109/ICSE.2007.37",
       "relevance": "Randoop — classic feedback-directed random test generation tool, foundational SBST baseline."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "A working Java test generation tool with an artifact link, but requires API calls to GPT-3.5 and is specific to complex methods."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that existing LLM tools perform worse than Evosuite on complex methods is mildly surprising, but the decomposition approach itself is expected."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward technical contribution."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Anonymous artifact link provided but unclear how easily the tool can be run by others."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from PKU (reputable but not headline-grabbing); uses GPT-3.5 but is not about a famous product."
     }
   }
 }