ai-research-survey

scan.json (30296B)

---

 {
   "paper": {
     "title": "On the Evaluation of Large Language Models in Unit Test Generation",
     "authors": [
       "Lin Yang",
       "Chen Yang",
       "Shutao Gao",
       "Weijing Wang",
       "Bo Wang",
       "Qihao Zhu",
       "Xiao Chu",
       "Jianyi Zhou",
       "Guangtai Liang",
       "Qianxiang Wang",
       "Junjie Chen"
     ],
     "year": 2024,
     "venue": "ASE '24",
     "arxiv_id": "2406.18181",
     "doi": "10.1145/3691620.3695529"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Open-source LLMs (CodeLlama, DeepSeek-Coder variants from 7B to 34B) all underperform traditional EvoSuite in unit test coverage, primarily due to 34-62% syntactically invalid test generation caused by LLM hallucination. Prompt design significantly affects effectiveness: aligning description style with training data and managing code feature inclusion to balance context vs. generation space matters more than model scale. Chain-of-Thought helps only models with strong code comprehension (DeepSeek-Coder), while RAG adapted from code generation consistently hurts all models. Defect detection is weak: on average 87% of defects lack any valid generated tests, and among testable defects, 75% go undetected mainly due to missing specific trigger inputs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.6 states 'All of our code and data are available at our project homepage [5]' linking to https://github.com/LeonYang95/LLM4UT."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use the publicly available Defects4J 2.0 benchmark and claim all data is available at their project homepage [5]."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.6 specifies 'Ubuntu 18.04 LTS, Intel Xeon Gold 6240C CPU, 512GB RAM, and eight NVIDIA A100 GPUs' with 'PyTorch 2.0.0 and transformers 4.34.1, and used VLLM libraries.'"
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Code and data released at project homepage [5] on GitHub. The paper describes the experimental pipeline in sufficient detail (Section 2.6) and the repository presumably contains runnable scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results (Tables 1-6) report only point estimates (CSR, CovL, CovB percentages) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Wilcoxon rank sum tests with significance level 0.05 throughout (Sections 3.1-3.3), reporting p-values to assess statistical significance of differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports 'Rank-biserial correlation scores to show the effect size' with a threshold of >0.3 for meaningful differences, following Cohen (2013)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "778 focal methods from 413 defects across 17 projects are used. Section 4 states 'our experimental design balances conclusion generalizability and evaluation costs well' but no power analysis or formal sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for determinism, producing single-run results. No variance, standard deviation, or spread measures are reported across experimental runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against EvoSuite (traditional technique) and GPT-4 (commercial LLM) as reference baselines in Table 4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "GPT-4 was the state-of-the-art commercial LLM at the time of study. EvoSuite is the widely-used standard baseline for automated test generation. Both are appropriate."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 performs extensive ablation on prompt design, removing one code feature at a time (Tables 2-3) to measure each feature's contribution. RQ3 tests adding CoT and RAG."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Compilation Success Rate (CSR), Line Coverage (CovL), Branch Coverage (CovB), and Number of Detected Defects (NDD)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4: four authors with 4+ years Java experience manually analyzed and labeled undetected defects into three categories (Cohen's Kappa = 0.95). This constitutes human evaluation of the system's outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The best prompt design is selected in RQ1 using the same Defects4J data, then used for RQ2-4 evaluation on the same data. No separate held-out set for configuration selection vs. final evaluation."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (Tables 1-6), per prompt variant (Tables 2-3), per error type (Section 3.2), and per defect failure reason (Table 7). Per-project breakdowns are at the homepage."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 analyzes three main compilation error types (unresolved symbols 30.68%, parameter mismatch 17.25%, abstract instantiation 10.38%). Section 3.4 analyzes why defects are undetected (Table 7)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: RAG hurts all models (Table 5), CoT hurts CodeLlama models, FCm inclusion hurts coverage despite helping validity, all LLMs underperform EvoSuite."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about prompt design influence (Tables 1-3), open-source vs GPT-4 vs EvoSuite performance (Table 4), ICL limitations (Table 5), and defect detection weaknesses (Table 6-7) are all supported by corresponding results sections."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims (e.g., 'removing FCm improves coverage', 'CoT improves DeepSeek-Coder') are based on controlled ablation experiments with single-variable manipulation and statistical testing."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper consistently references '17 Java projects from Defects4J', 'five open-source LLMs', and acknowledges in Section 4 that results are limited to their specific benchmark and model selections. Findings are framed as specific to the tested setting."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: training data alignment for style sensitivity (Section 3.1), code comprehension ability differences for CoT results (Section 3.3), token space trade-offs for coverage effects (Section 3.1), and repetition issues for CL-7B anomaly (Section 3.2)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Metrics (CSR, line/branch coverage, defect detection count) directly measure what is claimed. No proxy gap exists — the paper measures test validity and coverage and reports them as such."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Open-source models are precisely named (CodeLlama-7B-Instruct, Phind-CodeLlama-34B-v2, DeepSeekCoder-6.7B-Instruct, etc.) but GPT-4 is referred to only as 'GPT-4' without a snapshot date or API version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.3 describes prompt design in detail (NL vs CL styles, code features), and the paper links to a GitHub repository [5] containing all code, which includes the prompt implementations."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only temperature=0 is reported (Section 4). Other key settings like max tokens, top-p, and generation length limits are not specified, despite the paper's own argument that token space affects output quality."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The approach is single-pass prompt→response with post-processing (AST extraction and compilation)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5-2.6 documents: selection of public patched methods as focal methods (778 from 413 defects), AST-based test extraction using tree-sitter, integration into test classes, and import resolution."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 'Threats to Validity' provides substantive discussion across internal, external, and construct validity dimensions."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 discusses specific threats: code review and testing of experimental scripts, model selection criteria based on HuggingFace leaderboard, inability to evaluate all code feature combinations, specific data leakage analysis comparing generated vs original tests, and plans to extend to GitBug-Java."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 states specific boundaries: 'we may not find the globally optimal setting for each studied LLM', acknowledges the benchmark is limited to Defects4J Java projects, and discusses plans to extend to more recent benchmarks like GitBug-Java."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.6: 'All of our code and data are available at our project homepage [5]' on GitHub, enabling access to raw experimental outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5 describes using Defects4J 2.0, selecting patched public methods as focal methods, resulting in 778 focal methods from 413 defects across 17 projects."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the standard Defects4J benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: focal method selection → prompt construction → LLM generation → AST-based test extraction → test class integration → import resolution → compilation → execution → coverage measurement via JaCoCo."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSFC grants (62322208, 62202040, 62232001, 12411530122) and CCF-Huawei Populus Grove Fund."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed. Four authors (Xiao Chu, Jianyi Zhou, Guangtai Liang, Qianxiang Wang) are from Huawei Cloud Computing Co. Ltd., and the first author completed this work during a Huawei internship."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The CCF-Huawei Populus Grove Fund is partially funded by Huawei, and four co-authors are Huawei employees. Huawei has commercial interest in LLM-based software engineering tools, making the funder non-independent."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used. The paper acknowledges 'a potential data leakage threat' but does not specify when models' training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: 'we compared LLM-generated unit tests with the original unit tests equipped by this benchmark. We found that there is no exact match between them, and even the number of LLM-generated unit tests (3.70 on average) is largely different with that of original unit tests (2.41).'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 acknowledges 'a potential data leakage threat' with Defects4J (published 2014, before all models' training). They perform a basic comparison analysis and note plans to extend to more recent benchmarks (GitBug-Java)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The manual analysis of defects is a labeling task by authors, not a human subjects study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No per-inference cost or latency is reported. GPT-4 API costs are not mentioned. No wall-clock time per generation is provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'around 3,000 NVIDIA A100 GPU-hours' for all experiments, and specifies hardware: 'four servers with eight NVIDIA A100 GPUs each.'"
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0 for determinism but no seed sensitivity analysis is performed. No discussion of whether other sources of randomness (e.g., VLLM batching) affect results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. Temperature=0 implies single deterministic runs, but this is never stated directly."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The ablation on prompt design explores a systematic set of variants, but no hyperparameter search budget (total configurations tried, compute spent on search) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes a systematic ablation approach: best description style per model is selected based on statistical significance, and best code feature set is selected from ablation results. The selection criterion is clearly stated."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many Wilcoxon rank sum tests are conducted across models, styles, and feature variants, but no correction for multiple comparisons (Bonferroni, Holm, etc.) is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors use original model weights and EvoSuite with default settings, but do not discuss self-comparison bias or acknowledge that their experimental framework choices could favor certain outcomes."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 7B to 34B parameters are compared without discussing compute cost per inference. GPT-4 uses substantially more compute than 7B models but this is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J is used without discussion of whether it adequately represents real-world unit test generation scenarios. No analysis of potential selection bias in the benchmark's projects or defect types."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. All models go through the same single-pass generation and post-processing pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 acknowledges 'a potential data leakage threat' and notes Defects4J was published before model training. They compare generated vs original tests as a basic temporal leakage check and plan to extend to more recent benchmarks."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether input features (code context in prompts) could leak test-relevant information beyond what a real developer would have. The evaluation setup is not analyzed for information leakage."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple focal methods come from the same 17 projects but no discussion of whether this creates non-independence in the results. Methods from the same project likely share code patterns."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 4: they compare LLM-generated tests with original Defects4J tests, finding 'no exact match' and different average counts (3.70 vs 2.41). This is a concrete, though limited, detection method."
       }
     }
   },
   "claims": [
     {
       "claim": "Prompt description style significantly affects LLM effectiveness: CodeLlama models perform better with natural language style, while DeepSeek-Coder models are robust to style choice.",
       "evidence": "Table 1 shows CL-7B and CL-13B achieve significantly higher CSR, CovL, and CovB with NL style (all six cases statistically significant). DC-7B, DC-33B, PD-34B show no significant difference in 7/9 cases. Root cause traced to training data alignment (Section 3.1).",
       "supported": "strong"
     },
     {
       "claim": "Including other methods in the focal class (FCm) helps syntactic validity but harms test coverage due to consuming prompt space.",
       "evidence": "Table 2: removing FCm decreases CSR by 2.14-14.92% (statistically significant for all models). Table 3: removing FCm increases coverage in almost all cases with statistical significance. Average generated tests increase from 3,654 to 5,434 after removal.",
       "supported": "strong"
     },
     {
       "claim": "All studied LLMs, including GPT-4, underperform traditional EvoSuite in test coverage.",
       "evidence": "Table 4: EvoSuite achieves 78.91% CovL and 76.59% CovB vs GPT-4's 40.43% CovL and 31.78% CovB. The gap is attributed to 34-62% syntactically invalid LLM-generated tests.",
       "supported": "strong"
     },
     {
       "claim": "Chain-of-Thought improves DeepSeek-Coder models but hurts CodeLlama models in unit test generation.",
       "evidence": "Table 5: CoT brings +2.72% CovL for DC-7B (statistically significant) but -3.04% for CL-7B and -6.45% for CL-13B (statistically significant). Attributed to differences in code comprehension ability.",
       "supported": "moderate"
     },
     {
       "claim": "RAG adapted from code generation consistently hurts all LLMs in unit test generation.",
       "evidence": "Table 5: all five open-source LLMs show decreased CovL (3.34-9.28%) and CovB (4.44-8.78%) with RAG. Cause: significant gap between retrieved tests (avg 12.10 LOC, 2.41 per method) and generated tests (avg 5.60 LOC, 6.94 per method).",
       "supported": "strong"
     },
     {
       "claim": "Defect detection ability of LLM-generated tests is weak: 87.13% of defects have no valid tests, and only 47.28% of testable defects are detected.",
       "evidence": "Table 6: NTD ranges from 28 to 65 out of 413 defects. NDD ranges from 12 to 39. Table 7 analyzes reasons: 74.99% of undetected defects due to missing specific defect-triggering inputs.",
       "supported": "strong"
     },
     {
       "claim": "LLM hallucination causes three main types of compilation errors: unresolved symbol (30.68%), parameter mismatch (17.25%), and abstract instantiation (10.38%).",
       "evidence": "Section 3.2 provides detailed breakdown of error types from analyzing invalid generated tests across all LLMs.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Huawei conflict of interest",
       "detail": "Four of eleven authors are from Huawei Cloud Computing, the first author completed the work during a Huawei internship, and the study is partially funded by CCF-Huawei Populus Grove Fund. Huawei has commercial interest in AI-powered software engineering tools. No competing interests statement is provided."
     },
     {
       "flag": "No held-out evaluation set",
       "detail": "Best prompt configurations are selected in RQ1 on the same Defects4J data used for evaluation in RQ2-4. This means configuration selection and final evaluation use identical data, inflating apparent performance of the 'best' settings."
     },
     {
       "flag": "Weak contamination analysis",
       "detail": "Data leakage analysis consists only of comparing generated test counts and checking for exact matches with original tests. This is insufficient — models could have memorized patterns, structures, or partial solutions without producing exact copies."
     },
     {
       "flag": "No multiple comparison correction",
       "detail": "Dozens of Wilcoxon rank sum tests are conducted across models, styles, features, and ICL methods without any correction for family-wise error rate, increasing the risk of false positives."
     }
   ],
   "cited_papers": [
     {
       "title": "No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation",
       "authors": ["Zhiqiang Yuan", "Yiling Lou", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "Directly relevant prior study evaluating ChatGPT for unit test generation with CoT, serving as key comparison point."
     },
     {
       "title": "ChatUniTest: a ChatGPT-based automated unit test generation tool",
       "authors": ["Zhuokui Xie", "Yinghao Chen", "Chen Zhi"],
       "year": 2023,
       "arxiv_id": "2305.04764",
       "relevance": "ChatGPT-based unit test generation tool with self-repair capability, evaluated in prior work on LLM test generation."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "relevance": "Empirical evaluation of GPT-3.5 for JavaScript unit test generation (TestPilot), directly comparable study design."
     },
     {
       "title": "Exploring the Effectiveness of Large Language Models in Generating Unit Tests",
       "authors": ["Mohammed Latif Siddiq", "Joanna C. S. Santos"],
       "year": 2023,
       "arxiv_id": "2305.00418",
       "relevance": "Prior study evaluating GPT-3.5 and Codex effectiveness in unit test generation."
     },
     {
       "title": "CODAMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K Lahiri"],
       "year": 2023,
       "relevance": "Combines evolutionary search (EvoSuite-style) with LLM code generation to overcome coverage plateaus."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex evaluation paper establishing LLM code generation benchmarks and methodology."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Foundation model paper for CodeLlama, one of the two model families evaluated in this study."
     },
     {
       "title": "EvoSuite: automatic test suite generation for object-oriented software",
       "authors": ["Gordon Fraser", "Andrea Arcuri"],
       "year": 2011,
       "relevance": "The primary traditional baseline for automated test generation used as comparison point in this study."
     },
     {
       "title": "Effective test generation using pre-trained Large Language Models and mutation testing",
       "authors": ["Arghavan Moradi Dakhel", "Amin Nikanjam"],
       "year": 2024,
       "relevance": "Combines LLMs with mutation testing for test generation, exploring code feature effects on prompt design."
     },
     {
       "title": "Enhancing LLM-based Test Generation for Hard-to-Cover Branches via Program Analysis",
       "authors": ["Chen Yang", "Junjie Chen", "Bin Lin"],
       "year": 2024,
       "arxiv_id": "2404.04966",
       "relevance": "TELPA system integrating LLMs with program analysis for test generation, from overlapping author team."
     },
     {
       "title": "Large Language Models for Software Engineering: Survey and Open Problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mark Harman"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLMs in software engineering covering testing, code generation, and open challenges."
     },
     {
       "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2210.14179",
       "relevance": "Evaluates LLMs for automated program repair using similar Defects4J benchmark, with data leakage discussion."
     },
     {
       "title": "Unit Test Case Generation with Transformers",
       "authors": ["Michele Tufano", "Dawn Drain", "Alexey Svyatkovskiy"],
       "year": 2020,
       "arxiv_id": "2009.05617",
       "relevance": "AthenaTest: early DL-based unit test generation using BART transformer, foundational prior work."
     },
     {
       "title": "Software Testing With Large Language Models: Survey, Landscape, and Vision",
       "authors": ["Junjie Wang", "Yuchao Huang", "Chunyang Chen"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based software testing including test generation, relevant to survey scope."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Provides actionable guidelines for prompt design and LLM selection for unit test generation, though not a ready-to-use tool."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Some mildly surprising findings (RAG hurts all models, EvoSuite still dominates) but these confirm growing skepticism rather than overturning beliefs."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or dramatic claims; straightforward empirical study."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code released on GitHub but it's an experimental framework, not a user-facing tool."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-4 as reference and published at ASE, but authors are not from a widely-recognized AI lab."
     }
   }
 }