ai-research-survey

scan.json (25991B)

---

 {
   "paper": {
     "title": "An Empirical Evaluation of Using Large Language Models for Automated Unit Test Generation",
     "authors": [
       "Max Schäfer",
       "Sarah Nadi",
       "Aryaz Eghbali",
       "Frank Tip"
     ],
     "year": 2023,
     "venue": "IEEE Transactions on Software Engineering (arXiv preprint)",
     "arxiv_id": "2302.06527"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'our approach in TESTPILOT, an LLM-based test generation tool for JavaScript that...is available as open-source software at https://github.com/githubnext/testpilot' (Introduction, contributions bullet)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The raw data and analysis for all our experiments can be found at https://doi.org/10.6084/m9.figshare.23653371' (Section 2 introduction)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions JavaScript, Node.js, Mocha, Istanbul/nyc, and a GitHub Actions Linux VM (2-core CPU, 7GB RAM) but does not provide a requirements file, Dockerfile, or detailed dependency specifications sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While data and code are released via figshare and GitHub, the paper itself does not include step-by-step reproduction instructions. The artifact link points to external data but no explicit reproduction guide is included in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports median values across 10 runs but does not report confidence intervals or error bars for main coverage results. Coverage figures in Table 2 are single-value medians."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses Wilcoxon paired rank-sum tests to compare TESTPILOT vs. Nessie coverage (Section 4.2, p-values 0.002 and 0.027), and Wilcoxon matched-pairs signed rank tests for ablation comparisons (Section 4.5)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Cliff's delta effect sizes for all major comparisons (e.g., 'large effect size, measured by Cliff's delta [55], of 0.493 for statement coverage and a medium one (0.431) for branch coverage' in Section 4.2, and multiple effect sizes in Section 4.5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 25 npm packages but does not justify why 25 packages were chosen or perform a power analysis. The selection criteria are described but not statistically justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'we run all experiments 10 times' and reports medians, but does not report standard deviation, IQR, or any spread measure alongside the coverage medians in the main tables."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares TESTPILOT against Nessie, 'the state-of-the-art JavaScript test generator' (RQ2, Section 4.2), and also compares against three different LLMs (RQ7, Section 4.7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Nessie was published at ICSE 2022 and is described as the current state-of-the-art feedback-directed random test generation technique for JavaScript; the comparison is contemporaneous with the 2023 paper."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ5 (Section 4.5) presents an ablation study disabling each of the four prompt refiners (FnBodyIncluder, DocCommentIncluder, SnippetIncluder, RetryWithError) one at a time, with statistical significance testing on the differences."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports statement coverage, branch coverage, percentage of passing tests, non-trivial assertions, and test similarity (maximum edit distance) as evaluation metrics across RQ1-RQ6."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes explicit claims about test readability and naturalness (e.g., 'tests look quite natural and similar to tests that a human developer might write' in Section 2.3, and readability comparisons with Nessie throughout). These readability/naturalness claims make human evaluation relevant, not 'clearly irrelevant to the claims' as the schema requires for applies=false. The paper itself acknowledges this: 'we do not formally assess the readability of these tests. In the future, it would be interesting to conduct user studies to assess the readability' (Section 5, Construct Validity). Since the paper could reasonably be expected to include human evaluation of test quality given its readability claims, applies=true with answer=false is correct."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 25 npm packages are clearly specified as the evaluation benchmark (Table 1), and there is no tuning performed on these packages; TESTPILOT generates tests at inference time without using any test examples from the packages as training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides per-package coverage results for all 25 npm packages, and Table 4 provides per-package LLM comparisons. Results are not just aggregate medians."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 (Section 4.4) analyzes failing tests by type: assertion errors, file-system errors, correctness errors, timeout errors, and other errors. Figure 7 shows the breakdown per package."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (RQ5) shows cases where TESTPILOT performs worse (e.g., Figure 9 shows function body inclusion confusing the model). Section 4.5 notes '394 cases (7.3%) the refined prompt was less effective than the original prompt.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All major abstract claims are supported by results: median 70.2% statement coverage (Table 2), 52.8% branch coverage (Table 2), Nessie comparison (Section 4.2), ablation findings (Section 4.5), and LLM comparison (Section 4.7) are all present in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (RQ5) makes causal claims that each prompt component contributes to effectiveness; this is supported by controlled single-variable ablations with statistical significance tests (Wilcoxon matched-pairs signed rank tests with effect sizes in Section 4.5)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly bounds its claims: 'while our technique is conceptually language-agnostic, our current implementation of TESTPILOT targets JavaScript, and thus we cannot generalize our results to other languages' (Section 5, External Validity)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 (Threats to Validity) discusses specific alternative explanations: internal validity threat from snippet matching heuristics, construct validity threat from non-trivial assertion definition, and external validity threat that performance may not generalize to proprietary code or other languages."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact model versions: 'gpt-3.5-turbo-0301' (Section 3.2), 'code-cushman-002' (referenced as OpenAI's older Codex model), and StarCoder (with HuggingFace reference [49]). The version gpt-3.5-turbo-0301 is a specific snapshot."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 1, 3, and 4 show the actual prompt templates and example prompts used in experiments, including the full prompt structure and real examples. The structure is shown in Figure 1 and concrete populated examples appear in Figures 3 and 4."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 states: 'sampling five completions of up to 100 tokens at temperature zero, with all other options at their default values.' StarCoder uses temperature 0.01. Key hyperparameters are reported."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 and Algorithms 1-2 describe TESTPILOT's architecture in detail: API Explorer, Documentation Miner, Prompt Generator, Test Validator, and Prompt Refiner components with their logic, including the retry-with-error feedback loop."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The documentation mining process is described in Section 2.1 (Documentation Miner), and the API exploration procedure is formalized in Algorithm 1. The paper describes how snippets are matched to functions via textual containment."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'THREATS TO VALIDITY' contains dedicated subsections on Internal Validity, Construct Validity, and External Validity with substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Threats are specific to this study: the snippet matching heuristic for same-name functions (internal validity), the low bar for non-trivial assertion definition and limitations of backwards slicing in JavaScript (construct validity), and dependence on 25 npm packages and potential non-generalizability to proprietary code (external validity)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'we cannot generalize our results to other languages' and notes performance may not generalize to 'proprietary code that was never seen in the LLM's training set' (Section 5, External Validity)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw data is available at https://doi.org/10.6084/m9.figshare.23653371, explicitly stated in the paper: 'The raw data and analysis for all our experiments can be found at' this URL."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes how the 25 npm packages were selected (first 10 from Nessie benchmark, 10 new GitHub-hosted packages from different domains, 5 GitLab-hosted packages). Selection criteria (popularity, domain diversity, documentation availability) are stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "This paper has no human participants. Data source is publicly available npm packages on GitHub and GitLab, not human subjects."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The test generation pipeline is documented through Algorithms 1 and 2, and the architecture description in Section 2. The deduplication of generated tests is also described (stripping comments, comparing normalized forms)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states funding sources: NSF grants CCF-1907727 and CCF-2307742 (F. Tip) and Canada Research Chairs Program and NSERC RGPIN-2017-04289 (S. Nadi)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are disclosed on the first page: M. Schäfer is with GitHub, UK; S. Nadi with University of Alberta; A. Eghbali with University of Stuttgart; F. Tip with Northeastern University. The acknowledgment also notes S. Nadi and F. Tip were sabbatical visitors at GitHub during this work."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "M. Schäfer is employed by GitHub, and the acknowledgment notes the other authors conducted this research while at GitHub. The paper evaluates a GitHub-hosted tool (TESTPILOT) and compares against competing approaches. GitHub has a financial interest in demonstrating LLM-based test generation effectiveness (related to GitHub Copilot)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests statement or declaration of financial interests (patents, equity). The acknowledgment mentions GitHub affiliation but does not formally declare whether any authors hold financial interests in the outcomes."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges that 'gpt3.5-turbo...was trained on GitHub repositories' and uses this as motivation for including GitLab packages, but does not state a specific training data cutoff date for any of the three LLMs evaluated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ6 (Section 4.6) is entirely dedicated to investigating memorization by measuring maximum similarity (normalized edit distance) between generated tests and existing tests. The paper explicitly addresses whether TESTPILOT reproduces memorized tests."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges 'since gpt3.5-turbo...was trained on GitHub repositories, we have to assume that all our subject packages...were part of the model's training set' (Section 3.2), and mitigates by including 5 GitLab packages not in GitHub training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study; it evaluates automated test generation on npm packages."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.7 reports wall-clock time: 'median time for TESTPILOT to generate tests for a given function using gpt3.5-turbo is 15s, and the median time to generate a complete test suite for a given package is 6m 55s.' Similar figures are given for StarCoder (24s/function, 10m 48s/package) and code-cushman-002 (11s/function, 4m 53s/package). API cost in dollars is not reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions tests were run on 'a standard GitHub Actions Linux VM with a 2-core CPU, 7GB of RAM, and 14GB of SSD disk space' (footnote 12) but does not state total API spend or GPU hours consumed for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "TESTPILOT achieves a median statement coverage of 70.2% and branch coverage of 52.8% on 25 npm packages using gpt3.5-turbo.",
       "evidence": "Table 2, Section 4.1 (RQ1). Results are medians over 10 runs across 25 packages. Per-package results range from 33.9% to 93.1% statement coverage.",
       "supported": "strong"
     },
     {
       "claim": "TESTPILOT significantly outperforms Nessie (state-of-the-art JavaScript test generator) in both statement coverage (70.2% vs. 51.3%) and branch coverage (52.8% vs. 25.6%).",
       "evidence": "Table 2, Section 4.2 (RQ2). Wilcoxon paired rank-sum tests: p=0.002 (statement) and p=0.027 (branch). Cliff's delta effect sizes: 0.493 (large) and 0.431 (medium). TESTPILOT outperforms Nessie on 17 of 24 comparable packages.",
       "supported": "strong"
     },
     {
       "claim": "All four prompt refiners contribute to generating more effective tests; disabling any refiner results in statistically significant performance degradation.",
       "evidence": "Section 4.5 (RQ5), Figure 8. Wilcoxon matched-pairs signed rank tests show statistically significant differences between full configuration and each ablated configuration. Largest effect: Cliff's delta 0.582 for FnBodyIncluder, 0.531 for DocCommentIncluder on passing tests.",
       "supported": "strong"
     },
     {
       "claim": "92.8% of TESTPILOT's generated tests have at most 50% similarity to existing tests (measured by normalized edit distance), with none being exact copies.",
       "evidence": "Section 4.6 (RQ6), Figure 10. Maximum similarity measured using normalized Levenshtein distance against all existing tests in each package.",
       "supported": "strong"
     },
     {
       "claim": "The approach generalizes across LLMs; code-cushman-002 achieves similar results to gpt3.5-turbo (68.2% median statement coverage), while StarCoder achieves lower but still competitive results (54.0% median statement coverage).",
       "evidence": "Table 4, Section 4.7 (RQ7). Wilcoxon tests show no significant difference between gpt3.5-turbo and code-cushman-002, but significant differences between StarCoder and both OpenAI models (p<0.05). Even StarCoder exceeds Nessie's 51.3%.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "TESTPILOT, an adaptive LLM-based test generation tool for JavaScript, achieves a median 70.2% statement coverage and 52.8% branch coverage on 25 npm packages using gpt3.5-turbo, significantly outperforming the state-of-the-art Nessie tool (51.3% and 25.6% respectively) with statistical significance. Ablation studies show all four prompt refiners contribute meaningfully to test effectiveness, and contamination analysis finds that 92.8% of generated tests have at most 50% similarity to existing tests with no exact copies. The approach generalizes across multiple LLMs (gpt3.5-turbo, code-cushman-002, StarCoder), with even the open-source StarCoder model matching or exceeding Nessie's coverage.",
   "red_flags": [
     {
       "flag": "GitHub affiliation conflict",
       "detail": "The lead author (M. Schäfer) is employed by GitHub, the other authors conducted the research as sabbatical visitors or interns at GitHub, and the tool (TESTPILOT) is hosted on GitHub Next. GitHub has a commercial interest in demonstrating LLM-based code tooling effectiveness (related to GitHub Copilot). No competing interests statement is provided despite this non-trivial affiliation conflict."
     },
     {
       "flag": "No API cost reported",
       "detail": "The evaluation queries gpt3.5-turbo, code-cushman-002, and StarCoder across 1,684 API functions on 25 packages, repeated 10 times each. No dollar cost for API calls is reported, making it difficult to assess the practical budget required to reproduce this work."
     },
     {
       "flag": "Variance not reported for main results",
       "detail": "While the paper runs experiments 10 times and reports medians, no spread measure (std dev, IQR, min/max) is reported alongside the coverage medians in Tables 2-4, making it impossible to assess result stability across runs."
     },
     {
       "flag": "Training cutoff not stated",
       "detail": "The paper does not state the training data cutoff for gpt3.5-turbo, code-cushman-002, or StarCoder, making it difficult to fully assess whether the benchmark packages were in training data. The GitLab mitigation partially addresses this for GitHub-hosted packages."
     }
   ],
   "cited_papers": [
     {
       "title": "Nessie: Automatically testing javascript APIs with asynchronous callbacks",
       "authors": [
         "E. Arteca",
         "S. Harner",
         "M. Pradel",
         "F. Tip"
       ],
       "year": 2022,
       "doi": "10.1145/3510003.3510106",
       "relevance": "State-of-the-art baseline JavaScript test generation tool used as primary comparison in the evaluation."
     },
     {
       "title": "CodaMOSA: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": [
         "C. Lemieux",
         "J. P. Inala",
         "S. K. Lahiri",
         "S. Sen"
       ],
       "year": 2023,
       "relevance": "Related work on using LLMs to assist search-based test generation for Python; directly compared to TESTPILOT in discussion."
     },
     {
       "title": "Code Generation Tools (Almost) for Free? A Study of Few-Shot, Pre-Trained Language Models on Code",
       "authors": [
         "P. Bareiß",
         "B. Souza",
         "M. d'Amorim",
         "M. Pradel"
       ],
       "year": 2022,
       "arxiv_id": "2206.01335",
       "relevance": "Prior work on few-shot LLM test generation for Java, the closest direct predecessor to TESTPILOT."
     },
     {
       "title": "Unit test case generation with transformers and focal context",
       "authors": [
         "M. Tufano",
         "D. Drain",
         "A. Svyatkovskiy",
         "S. K. Deng",
         "N. Sundaresan"
       ],
       "year": 2021,
       "relevance": "Prior work (AthenaTest) on fine-tuned BART model for test generation, compared against in the related work section."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": [
         "M. Chen",
         "J. Tworek",
         "H. Jun"
       ],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "OpenAI Codex paper — the foundational LLM for code that motivated the approach."
     },
     {
       "title": "Starcoder: A state-of-the-art LLM for code",
       "authors": [
         "HuggingFace"
       ],
       "year": 2023,
       "relevance": "One of the three LLMs evaluated in RQ7; open-source with documented training process."
     },
     {
       "title": "CodeT: Code Generation with Generated Tests",
       "authors": [
         "B. Chen",
         "F. Zhang",
         "A. Nguyen"
       ],
       "year": 2022,
       "arxiv_id": "2207.10397",
       "relevance": "Related LLM approach for code generation + test generation from problem descriptions."
     },
     {
       "title": "On Learning Meaningful Assert Statements for Unit Test Cases",
       "authors": [
         "C. Watson",
         "M. Tufano",
         "K. Moran",
         "G. Bavota",
         "D. Poshyvanyk"
       ],
       "year": 2020,
       "doi": "10.1145/3377811.3380429",
       "relevance": "ATLAS: prior work using LLMs to generate assert statements for existing test cases."
     },
     {
       "title": "TOGA: A neural method for test oracle generation",
       "authors": [
         "E. Dinella",
         "G. Ryan",
         "T. Mytkowicz",
         "S. K. Lahiri"
       ],
       "year": 2022,
       "doi": "10.1145/3510003.3510141",
       "relevance": "Neural approach for test oracle generation compared and discussed in related work."
     },
     {
       "title": "Neural software analysis",
       "authors": [
         "M. Pradel",
         "S. Chandra"
       ],
       "year": 2022,
       "doi": "10.1145/3460348",
       "relevance": "Survey of neural techniques for software engineering tasks; directly cited as framing for the field."
     },
     {
       "title": "Test generation for higher-order functions in dynamic languages",
       "authors": [
         "M. Selakovic",
         "M. Pradel",
         "R. Karim",
         "F. Tip"
       ],
       "year": 2018,
       "doi": "10.1145/3276531",
       "relevance": "Prior JavaScript test generation work (LambdaTester) by overlapping authors, context for dynamic API exploration."
     }
   ]
 }