ai-research-survey

scan.json (26960B)

---

 {
   "paper": {
     "title": "LLM for Test Script Generation and Migration: Challenges, Capabilities, and Opportunities",
     "authors": [
       "Shengcheng Yu",
       "Chunrong Fang",
       "Yuchen Ling",
       "Chentian Wu",
       "Zhenyu Chen"
     ],
     "year": 2023,
     "venue": "International Conference on Software Quality, Reliability and Security (QRS)",
     "arxiv_id": "2309.13574",
     "doi": "10.1109/QRS60937.2023.00029"
   },
   "scan_version": 3,
   "active_modules": [],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository or archive is provided. The generated test scripts, prompts, and experimental artifacts are not released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No datasets are released. The experimental subjects are publicly available apps (Outlook, QQ Mail, NetEase Mail, Fliggy, Ctrip, Mafengwo), but the actual prompts, ChatGPT responses, and generated scripts are not made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. The paper mentions using the OpenAI API with gpt-3.5-turbo and Appium but provides no version numbers, dependency lists, or setup details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions are provided. The paper describes the general experimental approach but lacks specific steps, configuration files, or scripts needed to reproduce the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No quantitative results are reported with confidence intervals or error bars. The paper's evaluation is entirely qualitative, with no numerical metrics reported for any experiment."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are used. The paper makes claims about LLM capabilities based on qualitative observation of a handful of experiments with no statistical analysis."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "No effect sizes are reported. The paper provides no quantitative measurements of success rates, completion rates, or any other metric that could establish effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The sample consists of 6 mobile apps with approximately 9 experiments for RQ1, 3 for RQ2, and 6 for RQ3. No justification is given for why these numbers are sufficient to support the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'Some prompts are repeated to see whether ChatGPT will generate responses with different scripts to eliminate the effect of randomness' (Section 3.2) but reports no variance or spread measures from these repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included. The paper does not compare ChatGPT's performance against any existing test generation or migration tools (e.g., CraftDroid, AppTestMigrator) despite discussing them in the related work."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No baselines are included at all, so contemporariness is moot. The paper discusses several contemporary tools in Section 6 (e.g., CraftDroid, AppTestMigrator) but never compares against them."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests two approaches for RQ1 (direct prompting and dialogue-based), but does not perform systematic ablation of components. No controlled removal of individual features to measure their contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions evaluating 'grammatical accuracy, semantic correctness, and practical applicability' (Section 4.1) but reports no quantitative metrics for any of these dimensions. All evaluation is qualitative narrative."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "The authors manually verify the generated scripts: 'their alignment with the predefined test operation process is manually confirmed' (Section 4.1), and scripts are executed on devices with manual checking of results across all three RQs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a qualitative case study with no training/testing split. There is no dataset or benchmark to hold out."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by RQ (scenario-based generation, cross-platform migration, cross-app migration) and further by individual scenarios (login, sending email, adding email account, flight search) and apps."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple failure cases are discussed in detail: deprecated API usage, pop-up handling failures, input box focus issues (Section 4.1), context memory overflow causing failure on 'adding email account' scenario, and inability to self-correct scripts (Section 4.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports several negative results: ChatGPT failed to self-terminate the testing process in the 'adding email account' scenario, failed to generate a relevant script in the second test of that scenario, and could not self-correct errors in cross-platform migration (Sections 4.1, 4.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract is relatively cautious, framing the work as an 'investigation' that 'aims to enhance software testing practices' and 'contribute to the understanding of LLMs' capabilities.' These aspirational/exploratory claims are broadly consistent with the qualitative findings reported."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'This approach significantly reduces manual intervention and enhances script generation efficiency' (Section 4.1) without any controlled comparison or quantitative measurement. No baseline is compared against to support the causal claim of reduction/enhancement."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'LLM' broadly but only gpt-3.5-turbo is tested. Claims about 'LLMs' capabilities are generalized from a single model. The paper tests only 6 apps in 2 categories (email, travel) but draws broad conclusions about mobile app test script generation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the observed results are discussed. For example, the paper does not consider whether the successes are due to the simplicity of the chosen scenarios (login is a well-documented flow) or whether similar results could be achieved with simpler template-based approaches."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 'grammatical accuracy, semantic correctness, and practical applicability' of individual scripts but frames results as demonstrating LLM 'capabilities' for test automation broadly. The gap between manually checking a few scripts and claiming general test automation capability is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we use the API provided by OpenAI with the gpt-3.5-turbo model' (Section 3.2) but provides no specific version or snapshot date. The gpt-3.5-turbo model changes over time."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompt templates are provided for each phase (general prompt, initiation, exploration, summarization) but all contain placeholder values (e.g., '{}' for device names, package names, element IDs). The actual filled-in prompts used in experiments are not provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters (temperature, top-p, max tokens, etc.) are reported for the gpt-3.5-turbo model calls."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The dialogue-based approach in RQ1 describes a three-phase scaffolding (initiation, exploration, summarization) with iterative feeding of XML page elements to ChatGPT and task-based decision making (Section 4.1). The workflow is described with prompt templates for each phase."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes manually acquiring 'comprehensive information of the whole testing process' (Section 4.1) but does not document how this information was extracted, filtered, or formatted before being used in prompts."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 'Challenges' serves as a limitations section, discussing four specific limitations: Context Memory, API Usage Randomness, Human Effort Requirement, and Limited Supported Test Events."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 discusses general limitations of LLMs (context memory, API randomness, human effort, limited test events) rather than specific threats to the validity of this study's findings. No discussion of threats like selection bias in app choice, non-representative scenarios, or generalizability limits of a single-model study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of what populations/settings are excluded or what claims the authors are not making. The title implies broad LLM applicability despite testing only one model on six apps."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. The actual ChatGPT conversation logs, generated scripts, and execution results are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the general approach (manually acquiring app information, constructing prompts, running experiments) but does not provide detailed data collection procedures such as exact dates, specific app versions tested, or complete interaction logs."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "App selection is described vaguely: 'carefully selected based on their popularity, usability, and the varying complexity of their features' (Section 3.3). No specific criteria, popularity metrics, or systematic selection process is documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No data pipeline documentation. The paper goes from describing the experimental setup to reporting qualitative observations without documenting intermediate steps, how many prompts were tried, or how observations were systematically recorded."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgment section: National Natural Science Foundation of China (62141215, 62272220, 62372228), Science, Technology and Innovation Commission of Shenzhen Municipality, and National Undergraduate Training Program for Innovation and Entrepreneurship."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with Nanjing University, State Key Laboratory for Novel Software Technology. They evaluate a third-party product (ChatGPT by OpenAI) with no apparent affiliation conflict."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are Chinese government research agencies and university programs with no financial stake in whether ChatGPT performs well at test script generation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on any benchmark. It is a qualitative case study testing ChatGPT's ability to generate test scripts for specific mobile apps in real time."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The experimental subjects are live mobile apps where ChatGPT generates scripts interactively, not a fixed test set."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used. The evaluation is qualitative, based on manual verification of generated scripts against specific mobile app scenarios."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The experimental subjects are mobile apps and LLM-generated test scripts."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency measurements are reported for any of the ChatGPT interactions across the experiments."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, hardware specifications, or total API spend is mentioned."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs have a strong capability in test script generation and migration tasks of different complexities when provided with sufficient information.",
       "evidence": "Qualitative observations across three RQs showing ChatGPT can generate syntactically correct Appium scripts for login, email, and flight booking scenarios on 6 apps (Sections 4.1-4.3). No quantitative success rates or metrics provided.",
       "supported": "weak"
     },
     {
       "claim": "ChatGPT can generate grammatically and syntactically correct Appium test scripts from detailed scenario descriptions.",
       "evidence": "In RQ1, 'All scripts are both grammatically and syntactically correct, and their alignment with the predefined test operation process is manually confirmed' (Section 4.1). Nine experiments across three apps with varying complexity.",
       "supported": "moderate"
     },
     {
       "claim": "The dialogue-based approach significantly reduces manual intervention and enhances script generation efficiency compared to providing all information at once.",
       "evidence": "The dialogue-based approach is described and tested on two scenarios (login, adding email account) on NetEase Mail (Section 4.1). No quantitative comparison of effort or efficiency between the two approaches.",
       "supported": "weak"
     },
     {
       "claim": "Context memory limitations cause ChatGPT to fail on complex scenarios with many GUI elements.",
       "evidence": "In the 'adding email account' scenario, ChatGPT 'loses track of the prior context and fails to remember the original task' when the page contained excessive clickable elements (Section 4.1). Demonstrated in two test instances.",
       "supported": "moderate"
     },
     {
       "claim": "Cross-app test script migration essentially transforms into a few-shot generation task rather than true migration.",
       "evidence": "In RQ3, the authors observe that 'similar scenarios in different apps share similar core business logic, but test procedures may vary greatly' and 'the migration task essentially transforms into a few-shot generation task that uses an existing test script as an example' (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "ChatGPT lacks intrinsic ability to self-correct or autonomously acquire information about target applications.",
       "evidence": "In cross-platform migration (Section 4.2), ChatGPT 'operates solely on the basis of human-provided information, blindly altering the old script without any inbuilt capacity for problem resolution.' When information was intentionally excluded, ChatGPT could not compensate.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "case-study",
     "qualitative"
   ],
   "key_findings": "ChatGPT (gpt-3.5-turbo) can generate syntactically correct Appium test scripts for mobile apps when provided with detailed element identifiers and scenario descriptions, but faces significant limitations including context memory overflow on complex scenarios, inconsistent API usage (including deprecated APIs), and inability to self-correct. Cross-platform migration works when sufficient difference information is provided, but cross-app migration effectively reduces to new script generation rather than true migration. The approach requires substantial human effort to provide input information and fix technical issues in generated scripts.",
   "red_flags": [
     {
       "flag": "No quantitative metrics",
       "detail": "The entire evaluation is qualitative narrative. No success rates, completion rates, correctness scores, or any other quantitative metric is reported despite testing ~18 experiments. Claims of 'strong capability' rest on anecdotal observation."
     },
     {
       "flag": "Very small sample size",
       "detail": "Only 6 apps tested across 2 categories (email, travel), with ~9 experiments for RQ1, ~3 for RQ2, and ~6 for RQ3. This is far too few to support general claims about LLM capabilities for mobile test automation."
     },
     {
       "flag": "Claims outrun evidence",
       "detail": "The paper claims 'LLMs have a strong capability' and frames the work as investigating 'LLM' capabilities broadly, but tests only gpt-3.5-turbo. The title generalizes to 'LLM' from a single model's qualitative results."
     },
     {
       "flag": "No baselines or comparisons",
       "detail": "Despite discussing several existing test generation and migration tools (CraftDroid, AppTestMigrator, Appium-based approaches) in Section 6, no comparison is made against any of them. There is no way to assess whether LLMs are better, worse, or equivalent to existing approaches."
     },
     {
       "flag": "Irreproducible",
       "detail": "No code, data, prompts, or ChatGPT conversation logs are released. The prompt templates contain only placeholders. Without specific app versions, API settings, or actual prompts, the experiments cannot be reproduced."
     }
   ],
   "cited_papers": [
     {
       "title": "Application of Large Language Models to Software Engineering Tasks: Opportunities, Risks, and Implications",
       "authors": ["I. Ozkaya"],
       "year": 2023,
       "relevance": "Directly surveys LLM applications in software engineering, including code generation, testing, and documentation."
     },
     {
       "title": "Teaching Large Language Models to Self-Debug",
       "authors": ["X. Chen", "M. Lin", "N. Schärli", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Proposes self-debugging with LLMs via few-shot demonstrations, relevant to LLM-based automated program repair."
     },
     {
       "title": "Large Language Models are Few-Shot Testers: Exploring LLM-Based General Bug Reproduction",
       "authors": ["S. Kang", "J. Yoon", "S. Yoo"],
       "year": 2022,
       "arxiv_id": "2209.11515",
       "relevance": "Introduces LIBRO, using LLMs to automate test generation from bug reports — directly relevant to LLM-based testing."
     },
     {
       "title": "Adaptive Test Generation Using a Large Language Model",
       "authors": ["M. Schäfer", "S. Nadi", "A. Eghbali", "F. Tip"],
       "year": 2023,
       "arxiv_id": "2302.06527",
       "relevance": "TestPilot uses LLMs for automatic unit test generation with adaptive prompting, directly relevant to LLM test generation capabilities."
     },
     {
       "title": "Prompting is All Your Need: Automated Android Bug Replay with Large Language Models",
       "authors": ["S. Feng", "C. Chen"],
       "year": 2023,
       "arxiv_id": "2306.01987",
       "relevance": "AdbGPT uses LLMs with prompt engineering for automated Android bug replay, closely related to LLM-based mobile testing."
     },
     {
       "title": "Fill in the Blank: Context-Aware Automated Text Input Generation for Mobile GUI Testing",
       "authors": ["Z. Liu", "C. Chen", "J. Wang", "X. Che", "Y. Huang", "J. Hu", "Q. Wang"],
       "year": 2022,
       "arxiv_id": "2212.04732",
       "relevance": "QTypist uses pre-trained LLMs to generate semantic input text for mobile GUI testing, directly relevant to LLM-aided mobile testing."
     },
     {
       "title": "ChatGPT: A Study on Its Utility for Ubiquitous Software Engineering Tasks",
       "authors": ["G. Sridhara", "S. Mazumdar"],
       "year": 2023,
       "arxiv_id": "2305.16837",
       "relevance": "Evaluates ChatGPT on various software engineering tasks, providing baseline for LLM capability assessment."
     },
     {
       "title": "Is ChatGPT the Ultimate Programming Assistant–How Far Is It?",
       "authors": ["H. Tian", "W. Lu", "T. O. Li", "X. Tang", "S.-C. Cheung", "J. Klein", "T. F. Bissyandé"],
       "year": 2023,
       "arxiv_id": "2304.11938",
       "relevance": "Evaluates ChatGPT as a programming assistant, relevant to understanding LLM capabilities in software engineering."
     },
     {
       "title": "Prompt Sapper: LLM-Empowered Software Engineering Infrastructure for AI-Native Services",
       "authors": ["Z. Xing", "Q. Huang", "Y. Cheng", "L. Zhu", "Q. Lu", "X. Xu"],
       "year": 2023,
       "arxiv_id": "2306.02230",
       "relevance": "Proposes LLM-powered infrastructure for software engineering, relevant to agentic AI programming tools."
     },
     {
       "title": "ChatGPT Prompt Patterns for Improving Code Quality, Refactoring, Requirements Elicitation, and Software Design",
       "authors": ["J. White", "S. Hays", "Q. Fu", "J. Spencer-Smith", "D. C. Schmidt"],
       "year": 2023,
       "arxiv_id": "2303.07839",
       "relevance": "Catalogues prompt engineering patterns for SE tasks, relevant to LLM-based software engineering methodology."
     },
     {
       "title": "Test Transfer Across Mobile Apps Through Semantic Mapping",
       "authors": ["J.-W. Lin", "R. Jabbarvand", "S. Malek"],
       "year": 2019,
       "relevance": "CraftDroid performs cross-app test migration via semantic mapping — a key baseline for the test migration task this paper investigates."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans", "M. Bosma", "F. Xia", "E. Chi", "Q. V. Le", "D. Zhou"],
       "year": 2022,
       "arxiv_id": "2203.11171",
       "relevance": "Foundational work on chain-of-thought prompting, referenced as part of prompt engineering advancement enabling LLM capabilities."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Explores a practical problem (mobile test automation) but provides no usable tool, framework, or replicable methodology."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms expected findings — LLMs can generate some code but have limitations with context and API correctness."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns raised."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo released."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses ChatGPT which is widely recognized, but the paper itself is from an academic lab without high brand recognition."
     }
   }
 }