ai-research-survey

scan.json (26454B)

---

 {
   "paper": {
     "title": "Automated Unit Test Improvement using Large Language Models at Meta",
     "authors": [
       "Nadia Alshahwan",
       "Jubin Chheda",
       "Anastasia Finegenova",
       "Beliz Gokkaya",
       "Mark Harman",
       "Inna Harper",
       "Alexandru Marginean",
       "Shubho Sengupta",
       "Eddy Wang"
     ],
     "year": 2024,
     "venue": "FSE 2024 (32nd ACM Symposium on the Foundations of Software Engineering)",
     "arxiv_id": "2402.09171",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code or repository link is provided. The tool is an internal Meta system and the paper explicitly states details are 'commercially sensitive.'"
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset is released. All experiments were conducted on internal Meta codebases (Instagram, Facebook) and no data is made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or infrastructure details are provided. The LLMs used are internal and unnamed ('LLM1' and 'LLM2')."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The tool operates on internal Meta infrastructure with internal LLMs, and no steps are given to recreate the setup."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. Results are presented as point estimates (e.g., '75% of test classes had at least one new test case that builds correctly') without uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between LLM1 vs. LLM2, different temperatures, and different prompts are made based solely on raw success rate numbers without any statistical testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports raw percentages (e.g., 25% coverage improvement rate, 73% acceptance rate), these are descriptive counts rather than standardized effect sizes. The paper notes 'the effect size was low' for temperature comparisons (Section 3.3) but does not quantify it."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Sample sizes are reported (86 test classes for evaluation, 1,979 test classes for deployment) but no justification is given for why these sizes are adequate, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. The paper reports medians (e.g., 'median number of lines of code added by a TestGen-LLM test in the test-a-thon was 2.5') without any spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares TestGen-LLM performance to human engineers at the test-a-thon (Table 1), and contextualizes results against prior work such as SapFix acceptance rates (50% vs. 73%), Siddiq et al.'s coverage results, and other LLM test generation tools."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparisons reference contemporary work from 2023 including Nie et al., Yuan et al., Siddiq et al., and Schafer et al. The related work section discusses recent LLM-based test generation literature."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 evaluates the contribution of individual components: two LLMs (LLM1, LLM2), four prompt strategies, and temperature settings. Results show unique contributions of each prompt and LLM combination."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: build success rate (75%), pass rate (57%), coverage improvement rate (25%), acceptance rate by engineers (73%), and per-test-case success rates."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Engineers reviewed TestGen-LLM's generated diffs as part of the deployment. The 73% acceptance rate reflects human evaluation of the system's outputs. Additionally, the paper reports manual verification that generated tests covered valid corner cases (Section 3.2.1)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "No held-out test set is used. The evaluation in Section 3.3 was conducted on 86 Kotlin components, but there is no separation of tuning and evaluation sets. Tuning decisions (default temperature, LLM, prompt) were made on the same data used to report results."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by platform (Facebook vs. Instagram, Tables 3 and 6), by LLM (LLM1 vs. LLM2, Table 5), by temperature (Table 4), and by prompt strategy (Section 3.3)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 discusses the 4 rejected diffs and their reasons: trivial methods, single responsibility violations, missing assertions. Section 5 discusses LLM 'self-plagiarism', TODO-only tests, and other failure modes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the statement_to_complete prompt 'failed to add any unique test cases' for LLM1 (Section 3.3). It also notes the low success rate per test case (~4%), and that only 25% of classes gained coverage despite 57% passing."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims (75% build, 57% pass, 25% coverage increase, 11.5% class improvement, 73% acceptance) are supported by data in Sections 3.3 and 4. The abstract claim of '11.5%' is slightly higher than the '10%' reported in Section 4, but both figures appear in the paper (10% overall from Section 4, 11.5% from combined test-a-thon data)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit causal claims such as the ensemble approach producing better results, temperature zero being optimal, and LLM2 outperforming LLM1. However, deployment mode results are confounded by incremental coverage accumulation (as the paper itself acknowledges in Section 4), and no controlled experiments isolate these factors."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated Unit Test Improvement using Large Language Models at Meta' but the abstract claims 'the first report on industrial scale deployment of LLM-generated code.' The results are specific to Kotlin test classes on Instagram and Facebook with two internal Meta LLMs, yet the paper frames contributions in terms of 'Assured LLMSE' as a general paradigm without adequately bounding generalizability to other languages, systems, or LLMs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper acknowledges confounding factors in deployment results (Section 4): incremental coverage accumulation making later trials harder, the test-a-thon context potentially inflating acceptance, and different sample sizes for different configurations making comparisons unreliable. The paper explicitly warns against over-interpreting temperature=0.4 results due to confounds."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The LLMs are referred to only as 'LLM1' and 'LLM2' with no version, size, or architecture details. The paper states: 'Because details are commercially sensitive (and not relevant to this paper), we do not give details of the two LLMs' (footnote 1)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 provides the four prompt templates with their full text. These include placeholders ({existing_test_class}, {class_under_test}) which are filled with actual code — the fill values are the test classes and classes under test, which are deterministic inputs rather than hand-crafted content."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings are reported and explored (Table 4, temperatures 0.0 to 0.9). The default temperature of 0.0 is stated. Section 3.3 describes the hyperparameter sweep."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The three-stage filtration pipeline (builds → passes → improves coverage) is described in detail in Section 2, including the flakiness filter (5 repeated executions), coverage measurement, and the pre/post-processing steps. Figure 1 depicts the architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes how test classes were selected: Section 3.2 explains classes targeted were those subject to recent refactoring, Section 3.3 identifies 86 Kotlin components (31 Stories, 55 Reels). The filtration pipeline stages are documented with counts (Figure 2 Sankey diagram)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper has a 'Future Work and Open Problems' section (Section 7) but this discusses research directions, not study limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are systematically discussed. While the paper mentions specific confounds in Section 4 (incremental coverage making comparisons unfair), these are scattered observations rather than a structured discussion of validity threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to Kotlin, to Meta's internal LLMs, or to the specific test-a-thon context. The general framing as 'Assured LLMSE' suggests broad applicability without stating limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. All results come from internal Meta deployments and no supplementary data is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection is described: telemetry logging of every execution (Section 1), specific date ranges (Oct 29 - Dec 29, 2023), and which products/platforms were targeted (Instagram Reels, Stories, Facebook)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited for a study. The engineers were doing their normal jobs at test-a-thons, not participating in a research study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The filtration pipeline from raw LLM generation to final accepted tests is well-documented: generates → builds filter → passes filter → coverage filter → human review. Figure 2 provides a Sankey diagram with percentages at each stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is explicitly disclosed. All authors are from Meta, which implicitly funded the work, but no formal funding acknowledgment is present."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are listed as Meta Platforms Inc. employees. The paper title includes 'at Meta' making the affiliation prominent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Meta employees are evaluating a Meta-internal tool (TestGen-LLM) deployed on Meta products. Meta has a direct interest in the tool's success. The funder (Meta) is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for either LLM1 or LLM2. The LLMs are internal Meta models and no details about their training data are provided."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the test classes used in evaluation were part of the LLMs' training data. Since these are internal Meta LLMs likely trained on internal Meta code, overlap is plausible but not addressed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation is on internal Meta code with internal Meta LLMs. The possibility that these LLMs were trained on the very test classes they are asked to extend is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "This is not a human subjects study. Engineers participated in test-a-thons as part of their normal work, not as research participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Engineers were doing their regular work evaluating code diffs."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. The 36 engineers in the test-a-thon were performing their normal work duties."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Engineers were part of existing test-a-thon teams."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. No experimental conditions were assigned to participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study. Engineers were informed they were receiving LLM-generated diffs."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "Not a human subjects study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported. The number of trials is given (e.g., 32,531 total) but not the cost per trial or total compute consumed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, or hardware specifications are stated. The paper runs tens of thousands of LLM inference trials plus repeated test executions without quantifying the compute cost."
       }
     }
   },
   "claims": [
     {
       "claim": "75% of TestGen-LLM's test cases built correctly, 57% passed reliably, and 25% increased coverage on Instagram Reels and Stories.",
       "evidence": "Section 3.3 reports results over 86 Kotlin components (31 Stories, 55 Reels). Figure 2 Sankey diagram depicts the filtration funnel.",
       "supported": "strong"
     },
     {
       "claim": "TestGen-LLM improved 10% (abstract says 11.5%) of all test classes to which it was applied across the three test-a-thons.",
       "evidence": "Section 4 states '196 test classes were successfully improved, while the TestGen-LLM tool was applied to a total of 1,979 test classes' yielding approximately 10%.",
       "supported": "strong"
     },
     {
       "claim": "73% of TestGen-LLM's test improvements were accepted by developers for production deployment.",
       "evidence": "Section 4 states this figure. Breakdown across test-a-thons: Instagram Dec had 36/42 accepted; Facebook Dec had 144/280 accepted.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first report on industrial scale deployment of LLM-generated code with assured improvement guarantees.",
       "evidence": "Stated in the abstract and conclusions. No counter-evidence is presented, but the claim of 'first' is inherently hard to verify.",
       "supported": "weak"
     },
     {
       "claim": "Temperature zero produces the best default results for test generation.",
       "evidence": "Section 3.3 and Table 4. However, the paper itself cautions that sample sizes are highly unequal (30,483 trials at temp 0 vs. 334 at temp 0.4) and confounding factors exist.",
       "supported": "weak"
     },
     {
       "claim": "Each LLM and prompt strategy makes a unique contribution in the ensemble approach.",
       "evidence": "Section 3.3 shows that 3 of 4 prompts for LLM1 and all 4 for LLM2 contributed unique test cases not found by other combinations.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "case-study",
     "benchmark-eval"
   ],
   "key_findings": "Meta's TestGen-LLM tool uses an ensemble of LLMs with a three-stage filtration pipeline (builds, passes, improves coverage) to extend existing Kotlin unit test classes. On 86 Instagram components, 75% of generated tests built, 57% passed reliably, and 25% increased coverage. Deployed across three test-a-thons on Instagram and Facebook, it improved 10% of 1,979 targeted test classes, with 73% of recommendations accepted by engineers. The paper demonstrates that combining multiple LLMs and prompt strategies in an ensemble with automated quality filters can produce production-ready test improvements at industrial scale.",
   "red_flags": [
     {
       "flag": "Company evaluating its own product",
       "detail": "All 9 authors are Meta employees evaluating TestGen-LLM, an internal Meta tool, deployed on internal Meta codebases using internal Meta LLMs. There is a strong institutional interest in positive results. No external evaluation or independent replication is present."
     },
     {
       "flag": "Opaque LLMs prevent independent assessment",
       "detail": "The two LLMs used ('LLM1' and 'LLM2') are internal Meta models with no disclosed architecture, size, training data, or version information. This makes the results impossible to reproduce or verify, and makes it impossible to assess data contamination."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are reported as point estimates without confidence intervals, error bars, significance tests, or variance measures. With 32,531 total trials, statistical analysis would be straightforward but is absent."
     },
     {
       "flag": "Confounded deployment results acknowledged but not mitigated",
       "detail": "The paper acknowledges that deployment-mode results are confounded by incremental coverage accumulation (Section 4), making comparisons between temperatures, LLMs, and prompts unreliable. Despite this acknowledgment, the confounded numbers are still presented as results."
     },
     {
       "flag": "No limitations or threats-to-validity section",
       "detail": "For a top-venue empirical paper, the absence of a dedicated limitations section is notable. Scope boundaries are not stated, and generalization claims (Assured LLMSE as a paradigm) extend well beyond the tested setting."
     },
     {
       "flag": "Acceptance rate may be inflated by test-a-thon context",
       "detail": "Engineers were in test-a-thon mode (focused on writing tests) and in some cases pre-warned about LLM-generated diffs. The 73% acceptance rate may not reflect acceptance in normal engineering workflows. The paper partially acknowledges this for the Instagram test-a-thon but not systematically."
     }
   ],
   "cited_papers": [
     {
       "title": "Large Language Models for Software Engineering: Survey and Open Problems",
       "authors": ["Angela Fan", "Beliz Gokkaya", "Mitya Lyubarskiy", "Mark Harman", "Shubho Sengupta", "Shin Yoo", "Jie Zhang"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLM applications in software engineering, covering test generation and other tasks relevant to the survey scope."
     },
     {
       "title": "Software Testing with Large Language Model: Survey, Landscape, and Vision",
       "authors": ["Junjie Wang", "Yuchao Huang", "Chunyang Chen", "Zhe Liu", "Song Wang", "Qing Wang"],
       "year": 2023,
       "arxiv_id": "2307.07221",
       "relevance": "Literature review of 102 papers on LLM-based testing, debugging, and repair — directly relevant to understanding the landscape of AI-assisted testing research."
     },
     {
       "title": "Exploring the Effectiveness of Large Language Models in Generating Unit Tests",
       "authors": ["Mohammed Latif Siddiq", "Joanna C. S. Santos", "Ridwanul Hasan Tanvir", "Noshin Ulfat", "Fahmid Al Rifat", "Vinicius Carvalho Lopes"],
       "year": 2023,
       "arxiv_id": "2305.00418",
       "relevance": "Evaluates LLM unit test generation effectiveness with CodeX on HumanEval and EvoSuite benchmarks — a baseline comparison point for test generation quality."
     },
     {
       "title": "No More Manual Tests? Evaluating and Improving ChatGPT for Unit Test Generation",
       "authors": ["Zhiqiang Yuan", "Yiling Lou", "Mingwei Liu", "Shiji Ding", "Kaixin Wang", "Yixuan Chen", "Xin Peng"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "Evaluates ChatGPT's unit test generation with prompt engineering, reporting roughly one-third executable tests — a key comparison point."
     },
     {
       "title": "Adaptive Test Generation Using a Large Language Model",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "Reports 70% statement-level coverage for LLM-generated tests on smaller systems, relevant for comparing LLM test generation effectiveness."
     },
     {
       "title": "CODAMOSA: Escaping Coverage Plateaus in Test Generation with Pre-trained Large Language Models",
       "authors": ["Caroline Lemieux", "Jeevana Priya Inala", "Shuvendu K Lahiri", "Siddhartha Sen"],
       "year": 2023,
       "relevance": "Hybrid approach combining SBST with LLMs for test generation — directly relevant to understanding ensemble/hybrid testing methodologies."
     },
     {
       "title": "SapFix: Automated End-to-End Repair at Scale",
       "authors": ["Alexandru Marginean", "Johannes Bader", "Satish Chandra", "Mark Harman", "Yue Jia", "Ke Mao", "Alexander Mols", "Andrew Scott"],
       "year": 2019,
       "relevance": "Prior Meta deployment of automated program repair with 50% acceptance rate, providing a direct comparison point for industrial deployment of AI-assisted code generation."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen et al."],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces CodeX and HumanEval benchmark, foundational to LLM code generation evaluation."
     },
     {
       "title": "Assured LLM-Based Software Engineering",
       "authors": ["Nadia Alshahwan", "Mark Harman", "Alexandru Marginean", "Shubho Sengupta", "Eddy Wang"],
       "year": 2024,
       "relevance": "Companion paper defining the Assured LLMSE paradigm that TestGen-LLM instantiates — central to the methodological framework being evaluated."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière et al."],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Meta's open-source code LLM, relevant to understanding the landscape of code-generation models evaluated in the survey."
     },
     {
       "title": "Effective Test Generation Using Pre-trained Large Language Models and Mutation Testing",
       "authors": ["Arghavan Moradi Dakhel", "Amin Nikanjam", "Vahid Majdinasab", "Foutse Khomh", "Michel C Desmarais"],
       "year": 2023,
       "relevance": "Hybridizes LLMs with mutation testing for test generation — relevant to understanding complementary approaches to improving LLM-generated tests."
     },
     {
       "title": "Learning Deep Semantics for Test Completion",
       "authors": ["Pengyu Nie", "Rahul Banerjee", "Junyi Jessy Li", "Raymond J. Mooney", "Milos Gligoric"],
       "year": 2023,
       "arxiv_id": "2302.10166",
       "relevance": "Reports 29% executable tests with TeCo, providing a baseline for LLM test generation quality comparison."
     }
   ]
 }