ai-research-survey

scan.json (30360B)

---

 {
   "paper": {
     "title": "A System for Automated Unit Test Generation Using Large Language Models and Assessment of Generated Test Suites",
     "authors": [
       "Andrea Lops",
       "Fedelucio Narducci",
       "Azzurra Ragone",
       "Michelantonio Trizio",
       "Claudio Bartolini"
     ],
     "year": 2024,
     "venue": "International Conference on Software Testing, Verification and Validation Workshops",
     "arxiv_id": "2408.07846",
     "doi": "10.1109/ICSTW64639.2025.10962454"
   },
   "scan_version": 3,
   "active_modules": [
     "experimental_rigor",
     "data_leakage"
   ],
   "methodology_tags": [
     "benchmark-eval",
     "case-study"
   ],
   "key_findings": "AGONETEST automates end-to-end unit test generation and evaluation for Java projects using LLMs. On 10 repositories (94 focal classes), approximately 75% of LLM-generated test classes compiled and ~34% passed all tests. GPT-4 achieved higher coverage than GPT-3.5-turbo across most metrics, but human-written tests consistently outperformed both models on mutation coverage (0.69 vs. 0.55 best LLM). Zero-shot prompting unexpectedly outperformed few-shot for GPT-4.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides an anonymous link (https://anonymous.4open.science/r/classes2test) for the CLASSES2TEST dataset, but no link to the AGONETEST system code is provided. Only the dataset is released, not the tool."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The CLASSES2TEST dataset is released via an anonymous repository link (footnote 1). The dataset extends METHODS2TEST with 147,473 test classes from 9,410 repositories."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using Python libraries (tiktoken, LiteLLM, ElementTree) and Java build tools but does not provide a reproducible environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The paper describes the system architecture and workflow but does not provide commands, scripts, or a README for reproducing the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables IV and V report only point estimates (e.g., 0.879 instruction coverage) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GPT-4 zero-shot yields the best results and that human tests outperform LLMs on mutation coverage, but no statistical significance tests are performed for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No formal effect sizes (Cohen's d, odds ratios, etc.) are reported. Tables show raw coverage values but the magnitude of differences between configurations is not formally characterized."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section V-A states 10 repos were 'chosen to be representative enough of the variability encountered in real-world projects, while ensuring that is tractable by our system in terms of scale.' This is a vague rationalization, not a statistical justification or power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Temperature is set to 0, implying single deterministic runs, but no discussion of variability across different projects or classes beyond aggregate averages."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Human-written tests from the CLASSES2TEST dataset serve as a baseline. Table V includes a 'human' row for comparison across all metrics."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The paper extensively discusses EvoSuite and other automated test generation tools (ChatTester, TestPilot, CEDAR) in Section II but does not include any of them as experimental baselines. Only human-written tests and two LLM configurations are compared."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is performed on AGONETEST components. The comparison of zero-shot vs. few-shot is a technique comparison, not an ablation of system components (e.g., automated context extraction, library integration)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five coverage metrics are used (instruction, branch, line, method, mutation coverage) plus 19 test smell indicators. Table V reports all coverage metrics; Table II includes test smells."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of generated test quality is performed. All evaluation is automated through JaCoCo, PiTest, and TsDetect."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 10 repositories are randomly selected from CLASSES2TEST, but there is no explicit held-out test set separation. The few-shot example comes from a separate repository (junit5-samples), but no dev/test split is discussed."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table II provides per-project, per-class breakdowns of metrics and test smells. Table IV and V break results by model and prompt technique."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section V-C discusses compilation failures and test failures in detail. Table IV quantifies rejection rates (30-38% test pass rate). Section VI-A1 identifies root causes: incorrect imports, syntax errors, calls to non-existent APIs."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that ~66% of generated tests were rejected (Table IV), human tests consistently outperform on mutation coverage (Table V), and few-shot surprisingly did not help GPT-4. Section VI acknowledges room for improvement."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'a scalable automated software system, a new dataset, and a detailed methodology for evaluating test quality.' The paper delivers the system description (Section III-IV), the dataset (CLASSES2TEST, Table I), and the evaluation methodology (Section V). Claims are appropriately scoped."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section VI states 'we found better results for zero-shot prompts for gpt-4 than for the few-shot one,' implying the prompting technique caused the difference. No statistical tests or controlled experimentation isolates this effect from confounds like example quality or interaction effects."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A explicitly states: 'its scope is limited to Java projects. This makes our findings hardly generalizable to different programming languages.' Section VII-B acknowledges the limited number of LLMs and prompts tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI discusses alternative explanations: the few-shot example may not be optimal, temperature settings restrict output diversity (Section VII-B), compilation failures may stem from LLM limitations rather than test design issues (Section VI-A1)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-D acknowledges: 'While we employed a comprehensive set of metrics and test smell indicators, these metrics alone may not fully capture the quality of the test suite.' This explicitly distinguishes measured proxies (coverage, smells) from the broader outcome (test quality)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Listing 1 specifies 'gpt-4-1106-preview' (a versioned model) but 'gpt-3.5-turbo' without a snapshot date. The gpt-3.5-turbo endpoint has changed behavior over time, making results non-reproducible without a pinned version."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates for both zero-shot and few-shot are provided in Listing 1, with variable placeholders ({{focal_class}}, {{testing_framework}}, {{java_version}}) whose values are deterministically extractable from the released CLASSES2TEST dataset with saved commit hashes."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature is explicitly set to 0 and the choice is justified in Section V-B: 'to increase the level of coherence in text generation (and to decrease the level of randomness) and make the diverse test suite generated comparable.' No other hyperparameters (top_p, max_tokens) are reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "AGONETEST uses a linear pipeline (prompt → LLM call → extract code → compile → evaluate) with no agentic scaffolding, retry loops, or iterative LLM interaction. The LLM is called once per focal class."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A documents the 4-step process for creating CLASSES2TEST from METHODS2TEST. Section V-C describes the filtering pipeline: generated tests → compilation check → green suite check → metric computation, with rejection counts in Table IV."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII 'Limitations' is a dedicated section with four subsections (Dataset and Generalization, Model and Prompt Variability, Compilation and Execution Failures, Evaluation Metrics)."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations are specific to this study: Java-only scope (VII-A), only two models tested (VII-B), temperature=0 may limit diversity (VII-B), 66% test rejection rate (VII-C), bias toward well-structured codebases in dataset selection (VII-A)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section VII-A: 'its scope is limited to Java projects.' VII-B: 'our initial experimental setup involved only two models... and two prompt types.' VII-B: 'results might vary significantly with newer models and advanced prompts.' Explicit about what was NOT tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The CLASSES2TEST dataset is available but the raw experimental outputs (individual test files, per-class metrics, CSV results) are not released. Only aggregate results are shown in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III-A describes the 4-step process for building CLASSES2TEST: extract repository references, select classes from METHODS2TEST, clone repositories and save commit hashes, map focal classes to test classes. Table I summarizes dataset characteristics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is the standard METHODS2TEST benchmark, which is a publicly available dataset of open-source Java repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented across Sections III-V: dataset construction (4 steps, Section III-A), configuration extraction (Section III-B), prompt creation (Section III-C), test generation (Section III-D), filtering of failing tests (Section V-C with counts in Table IV), and metric computation (Section III-E)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. One author is affiliated with Wideverse (a company), suggesting potential industry funding, but this is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Polytechnic University of Bari, Wideverse (company), University of Bari, and one Independent Researcher. The evaluated models are from OpenAI, with which the authors have no disclosed affiliation."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so funder independence cannot be assessed. One author is from Wideverse, a company, but no funding relationship is described."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for either GPT-4 or GPT-3.5-turbo. This is important since the test repositories are from GitHub and may be in the models' training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the GitHub repositories in CLASSES2TEST appeared in the training data of GPT-4 or GPT-3.5-turbo. The 9,410 repositories are public and likely included in OpenAI's training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "METHODS2TEST was published in 2022; GPT-4's training data extends well past this. The public GitHub repositories could have been seen during model training, including their test classes. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on code repositories."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or wall-clock time per test generation are reported. The paper mentions using tiktoken for token counting but does not report actual token usage or costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget is stated. The paper mentions scalability challenges (Section VI-A3) but does not quantify GPU hours, API spend, or hardware used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature is set to 0, implying deterministic outputs, but the paper does not discuss whether results are sensitive to this choice or report results across different settings."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state the number of experimental runs. Temperature=0 implies single deterministic runs, but this is not stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is performed. Temperature=0 is chosen without exploring alternatives. Section VII-B acknowledges 'Different temperature settings could yield more varied results, which were not explored in this study.'"
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All four configurations (2 models × 2 prompts) are reported in Table V without cherry-picking. The paper explicitly states 'this does not mean that gpt-4 is the best model for all scenarios, and this is neither the objective of our experiment' (Section VI)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 4 configurations and 5+ metrics with no statistical tests at all, let alone multiple comparison corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own AGONETEST system and implement the prompt strategies, but do not acknowledge or discuss self-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "GPT-4 and GPT-3.5-turbo have significantly different compute costs, but no cost-performance analysis is provided. Results are compared without considering the cost differential."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section VII-D briefly notes 'these metrics alone may not fully capture the quality of the test suite' but provides no substantive analysis of whether coverage metrics and test smells actually measure test quality as claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All model comparisons use the same AGONETEST pipeline with identical prompt templates and evaluation infrastructure. The scaffold is held constant across all configurations."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The GitHub repositories in CLASSES2TEST are public and many predate GPT-4's training cutoff. The models may have seen both the focal classes and their human-written tests during training. This is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The prompt provides the focal class to the LLM. If the LLM has memorized the corresponding test class from training data, the generated test is recall rather than generation. This possibility is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 9,410 GitHub repositories in CLASSES2TEST overlap with OpenAI's training data. Given these are popular open-source repos, high overlap is probable."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference, temporal splits, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "AGONETEST provides an end-to-end automated process for generating and evaluating test suites without human intervention.",
       "evidence": "System architecture described in Section III with modules for repository selection, configuration, prompt creation, test generation, and assessment. Section IV provides an end-to-end example. Section VI-RQ1 discusses automation degree.",
       "supported": "moderate"
     },
     {
       "claim": "Approximately 75% of LLM-generated test classes compile successfully.",
       "evidence": "Table IV shows compilation rates: gpt-3.5-turbo 68-69%, gpt-4 80.85%. Average across 4 configurations is ~75%.",
       "supported": "moderate"
     },
     {
       "claim": "Approximately 34% of generated test classes are green suite (all tests pass) with calculable mutation coverage.",
       "evidence": "Table IV shows pass rates: gpt-3.5-turbo 38-37%, gpt-4 31-30%. Rejected percentages range 62-70%.",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 outperforms GPT-3.5-turbo on most coverage metrics.",
       "evidence": "Table V shows GPT-4 zero-shot achieves higher instruction (0.879 vs 0.757), branch (0.777 vs 0.706), line (0.866 vs 0.777), and method coverage (0.855 vs 0.848). No statistical tests support this comparison.",
       "supported": "weak"
     },
     {
       "claim": "Human-written tests consistently outperform LLM-generated tests on mutation coverage.",
       "evidence": "Table V shows human mutation coverage of 0.691 vs best LLM at 0.547 (GPT-4 zero-shot and GPT-3.5 zero-shot). No statistical test performed.",
       "supported": "moderate"
     },
     {
       "claim": "Zero-shot prompting yields better results for GPT-4 than few-shot prompting.",
       "evidence": "Table V shows GPT-4 zero-shot outperforms GPT-4 few-shot on instruction (0.879 vs 0.753), line (0.866 vs 0.782), and mutation coverage (0.546 vs 0.462). Based on a single example for few-shot, no statistical tests.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny sample size",
       "detail": "Only 10 repositories with 94 focal classes are used for evaluation, and after filtering, results are based on as few as 29-36 green suite classes per configuration (Table IV). This is too small to draw reliable conclusions about LLM test generation quality."
     },
     {
       "flag": "No statistical tests",
       "detail": "All comparisons between models, prompt strategies, and human baselines are based solely on comparing raw numbers. No significance tests, confidence intervals, or effect sizes are reported for any claim."
     },
     {
       "flag": "Survivorship bias in evaluation",
       "detail": "Approximately 66% of generated tests are rejected (non-compiling or failing). Metrics are computed only on the surviving ~34%. This systematically inflates the apparent quality of LLM-generated tests by excluding failures."
     },
     {
       "flag": "Training data contamination risk",
       "detail": "The 9,410 GitHub repositories are public and likely present in GPT-4 and GPT-3.5-turbo training data. The models may have memorized test classes rather than generating them. This fundamental validity threat is not discussed."
     },
     {
       "flag": "Missing automated tool baselines",
       "detail": "Despite extensive discussion of EvoSuite, ChatTester, TestPilot, and other tools in Section II, none are included as experimental baselines. Only human-written tests are compared against."
     },
     {
       "flag": "Single few-shot example",
       "detail": "The few-shot prompt always uses the same example from junit5-samples (Section V-B). The poor few-shot results for GPT-4 may reflect the choice of example rather than a property of few-shot prompting, but this is not explored."
     }
   ],
   "cited_papers": [
     {
       "title": "EvoSuite: Automatic Test Suite Generation for Object-Oriented Software",
       "authors": ["G. Fraser", "A. Arcuri"],
       "year": 2011,
       "relevance": "Foundational search-based automated test generation tool for Java, the primary baseline approach that AGONETEST aims to improve upon."
     },
     {
       "title": "An Initial Investigation of ChatGPT Unit Test Generation Capability",
       "authors": ["V. Guilherme", "A. Vincenzi"],
       "year": 2023,
       "relevance": "Early empirical evaluation of ChatGPT for unit test generation, examining hyperparameter impact on test quality."
     },
     {
       "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
       "authors": ["M. Schäfer", "S. Nadi", "A. Eghbali", "F. Tip"],
       "year": 2023,
       "relevance": "Empirical evaluation of LLMs for automated unit test generation, one of the key prior works in this area."
     },
     {
       "title": "Using Large Language Models to Generate JUnit Tests: An Empirical Study",
       "authors": ["M. L. Siddiq", "J. C. Da Silva Santos", "R. H. Tanvir"],
       "year": 2024,
       "relevance": "Evaluation of LLMs for JUnit test generation with new metrics, directly comparable to AGONETEST's evaluation methodology."
     },
     {
       "title": "No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation",
       "authors": ["Z. Yuan", "Y. Lou", "M. Liu"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "ChatTester tool for evaluating and improving LLM-generated unit tests, a direct precursor to AGONETEST."
     },
     {
       "title": "ChatGPT vs SBST: A Comparative Assessment of Unit Test Suite Generation",
       "authors": ["Y. Tang", "Z. Liu", "Z. Zhou", "X. Luo"],
       "year": 2024,
       "relevance": "Direct comparison of ChatGPT vs search-based software testing for unit test generation."
     },
     {
       "title": "Methods2Test: A Dataset of Focal Methods Mapped to Test Cases",
       "authors": ["M. Tufano", "S. K. Deng", "N. Sundaresan", "A. Svyatkovskiy"],
       "year": 2022,
       "relevance": "The foundational dataset (780,944 method-test pairs from 9,410 repos) that CLASSES2TEST extends to class-level."
     },
     {
       "title": "Unit Test Case Generation with Transformers and Focal Context",
       "authors": ["M. Tufano", "D. Drain", "A. Svyatkovskiy"],
       "year": 2020,
       "arxiv_id": "2009.05617",
       "relevance": "AthenaTest: early transformer-based approach to unit test generation using fine-tuned BART model."
     },
     {
       "title": "Retrieval-Based Prompt Selection for Code-Related Few-Shot Learning",
       "authors": ["N. Nashid", "M. Sintaha", "A. Mesbah"],
       "year": 2023,
       "relevance": "CEDAR: retrieval-based prompt construction for code generation few-shot learning, relevant to prompt engineering for test generation."
     },
     {
       "title": "ChatUnitTest: A Framework for LLM-Based Test Generation",
       "authors": ["Y. Chen", "Z. Hu", "C. Zhi"],
       "year": 2024,
       "relevance": "LLM-based test generation framework, directly relevant to automated test generation infrastructure."
     },
     {
       "title": "Large Language Models Are Few-Shot Testers: Exploring LLM-Based General Bug Reproduction",
       "authors": ["S. Kang", "J. Yoon", "S. Yoo"],
       "year": 2023,
       "relevance": "Explores LLM-based bug reproduction and test generation, relevant to automated correction of generated test code."
     },
     {
       "title": "Large Language Models Are Zero-Shot Fuzzers: Fuzzing Deep-Learning Libraries via Large Language Models",
       "authors": ["Y. Deng", "C. S. Xia", "H. Peng"],
       "year": 2023,
       "relevance": "LLM-based fuzzing for software testing, demonstrating zero-shot test generation capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "AGONETEST addresses a real practitioner need (automated test generation), but the tool code is not released and the 34% green suite rate limits immediate usability."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expected patterns: GPT-4 beats GPT-3.5, human tests beat LLMs on mutation coverage, many generated tests don't compile."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No security or safety implications; the paper addresses software testing automation."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical evaluation of test generation approaches."
     },
     "demo_ability": {
       "score": 1,
       "justification": "The CLASSES2TEST dataset is available via anonymous link, but the AGONETEST tool code is not released for others to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-4 and GPT-3.5 (well-known models) but the paper and authors are from a regional university, not a major AI lab."
     }
   }
 }