ai-research-survey

scan.json (30447B)

---

 {
   "paper": {
     "title": "STELLAR: A Search-Based Testing Framework for Large Language Model Applications",
     "authors": [
       "Lev Sorokin",
       "Ivan Vasilev",
       "Ken E. Friedl",
       "Andrea Stocco"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2601.00497",
     "doi": "10.48550/arXiv.2601.00497"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Replication package released at https://github.com/ast-fortiss-tum/STELLAR (reference [35]), explicitly cited in Section IX: 'The pipeline used to obtain the results discussed in this work and the results are available in our replication package.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "They use publicly available datasets (BeaverTails [16], Yelp [51], MultiWOZ 2.2 [15]) and state results are in the replication package. NaviQA-II data is proprietary but NaviQA-I uses open-source data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions OpenSBT, NSGA-II, Azure, Ollama/HuggingFace, and ALL-MINILM-L6-V2 for embeddings, but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions in the paper. The replication package is referenced but the paper itself does not include commands to run or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Figures 2 and 3 show standard deviation bands for failure ratio over time (bottom plots), and box plots showing distribution spread for failure counts (top plots). Results averaged over 6 runs."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.F.2: 'we applied the Mann–Whitney U test [60] (α = 0.05) and quantified effect size using Vargha–Delaney A12 [61]. The improvements of STELLAR over all baselines are statistically significant, with large effect sizes.'"
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "They report Vargha-Delaney A12 effect sizes alongside significance tests, and provide context with relative improvements: '2.2 times higher failure rate' (SafeQA), '2.5 times higher failure rate' (overall), 'up to 4.3×' improvement."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "They run 6 repetitions per configuration but provide no justification for why 6 runs and no power analysis. The choice of 10 human raters is also unjustified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figures 2 and 3 show 'Mean ratio between failures found and in total generated test cases with standard deviation' across 6 runs. Box plots show distribution spread."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "They compare against Random Search (RS), ASTRAL (state-of-the-art automated LLM testing), and T-WISE (combinatorial 4-wise feature interaction) across all experiments."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ASTRAL is from ISSTA '25, a recent and directly relevant automated LLM testing approach. T-WISE is a standard combinatorial method. Both are appropriate and current baselines."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study removing individual STELLAR components (e.g., crossover vs. mutation, RAG-based generation, duplicate elimination, constraint rules). The system has multiple components but their individual contributions are not measured."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "They use multiple metrics: absolute failure count, failure ratio (failures/total tests), and cluster coverage for diversity (RQ2). NaviQA uses two fitness functions (f1 response quality, f2 content accuracy)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.E.2: 10 BMW participants rated 30 question-answer pairs for judge validation (300 annotations). Section VI: domain expert with 7 years experience evaluated failure types and answered structured questions about severity and novelty."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (population size, search time, mutation/crossover rates) were tuned via 'preliminary experiments' without clear separation from the final evaluation AUTs. No explicit train/validation/test separation for parameter tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results broken down per LLM model (Figures 2, 3), per case study (SafeQA, NaviQA-I, NaviQA-II), and per failure type (Table V with 9 failure categories for NaviQA-II)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI provides detailed qualitative evaluation of failure types in NaviQA-II (Table V). Nine failure types identified with examples: endpoint failure, incorrect rating, name misinterpretation, language misclassification, technical output, etc."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "They report that locally deployed LLMs (LLAMA 3, QWEN 2.5-7B, DEEPSEEK-V2-16B) had >90% failure rates on NaviQA and were excluded. ASTRAL only outperforms RS/T-WISE in 2 of 6 LLM configurations. RS/T-WISE occasionally match STELLAR's diversity."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'up to 4.3× (average 2.5×) more failures' are supported by results in Figures 2 and 3 and the statistical analysis. All key claims in the abstract are backed by the experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Comparative claims ('STELLAR detects more failures') are supported by controlled experiments: same testing budget, same LLMs, same evaluation criteria. Statistical significance tests with effect sizes provide adequate justification for the comparison."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "External validity section explicitly states: 'the scope is still limited. Generalization to other domains, larger systems, or additional LLM families should be made with caution.' The paper also notes budget constraints limited the number of models tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Threats to validity (Section V) discusses specific alternatives: threshold sensitivity may alter results, LLM non-determinism, API call duration variability, generator occasionally producing misaligned utterances, offline RAG configuration for ASTRAL vs. online."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures failure count and failure rate, and claims testing effectiveness. The measurements directly match the claims — no proxy gap exists. They additionally validate with diversity metrics and qualitative expert evaluation to triangulate."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models listed by marketing names only: GPT-4O, GPT-5-CHAT, DEEPSEEK-V3, MISTRAL 7B, DEEPSEEK-V2-16B, QWEN 2.5-7B, GPT-4O-MINI, DOLPHIN 3. No API version IDs, snapshot dates, or exact model identifiers provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 1 shows an 'illustrative' prompt template with placeholders ({{content}}, {{style}}, {{perturbation}}, {{rag_examples}}, {{examples}}). Full prompts referenced in replication package [35] but not in the paper. Template with unfilled placeholders does not count per schema."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I reports population size (20), search time (2h/3h), crossover threshold (0.7), mutation threshold (0.12/0.07), similarity threshold (0.8), number of fitness functions. Temperature values stated: 0/0 for SafeQA generator/judge, 0.2/0 for NaviQA."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "STELLAR is a search-based testing framework, not an agentic system. It uses LLMs for test generation and evaluation in a pipeline (no tool use, retry logic, memory, or agent scaffolding). The framework architecture is described via Algorithm 1."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Feature discretization is documented (Section III.A), numerical encoding explained, constraint rules described (Section III.B), duplicate elimination via cosine similarity with 0.8 threshold (Section III.F), embedding model specified (ALL-MINILM-L6-V2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section V 'Threats to Validity' contains three detailed subsections: Internal Validity, Construct Validity, and External Validity, with substantive discussion in each."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: threshold values may alter results (internal), LLM non-determinism despite repeated runs (internal), generator may produce misaligned utterances (internal), custom judge dimensions introduce subjectivity (construct), offline RAG for ASTRAL may differ from online (construct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity section: 'Although the systems and models differ in nature, the scope is still limited. Generalization to other domains, larger systems, or additional LLM families should be made with caution.' Future work explicitly lists unaddressed areas (multi-modal, multi-turn)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section IX: 'The pipeline used to obtain the results discussed in this work and the results are available in our replication package [35].' Results include generated test inputs and system outputs."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Test input generation pipeline fully described (Section III): feature sampling → encoding → constraint application → prompt generation → LLM generation → execution → evaluation. Judge validation procedure described with sampling from BeaverTails."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the 10-participant BMW judge validation study, only 'from BMW' is stated with no recruitment method, selection criteria, or potential bias discussion. The domain expert for qualitative evaluation is described only as having '7 years of working experience.'"
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 documents the full pipeline: random sampling → encoding → constraint application → prompt generation → LLM generation → AUT execution → fitness evaluation → evolutionary optimization → failure extraction. Each step is detailed in Sections III.A-III.G."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgements: 'This research was funded by the Bavarian Ministry of Economic Affairs, Regional Development and Energy, and by the BMW Group.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Three authors (Sorokin, Vasilev, Friedl) listed with BMW Group affiliation. Andrea Stocco listed with TU Munich and fortiss GmbH. Affiliations clearly stated in author block."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "BMW Group both funds the research and provides the industrial system under test (NaviQA-II). BMW has a commercial interest in the results — the funder is not independent of the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement. Three authors are BMW employees evaluating a BMW system. No patent or equity disclosures."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates a testing tool (STELLAR) rather than model knowledge on a benchmark. Models are tested for robustness to adversarial inputs, not for knowledge retrieval. Test inputs are dynamically generated, not drawn from a pre-existing benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "STELLAR generates new test inputs dynamically via evolutionary search, so there is no fixed test set that could overlap with model training data. The paper tests a tool/defense rather than evaluating model knowledge."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Test inputs are generated at runtime, not drawn from a pre-existing benchmark. The paper itself acknowledges data contamination as a limitation of static benchmarks (Section I) and positions STELLAR as an alternative to contamination-prone static evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the 10-participant judge validation study or the expert interview."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the human evaluation study involving 10 BMW participants and the domain expert."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The 10 participants are described only as 'from BMW.' No experience level, role, gender, age, or other demographics reported. The domain expert is described only as having '7 years of working experience in the testing of AI-enabled systems.'"
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria stated for selecting the 10 BMW participants. No screening process described."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "The human study is a simple annotation/rating task where all participants rated the same items. No experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "The human study is an annotation task evaluating response quality, not a comparative experiment requiring blinding between conditions."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No explicit reporting of how many participants started vs. finished. The total of 300 annotations (10 × 30) implies full completion, but attrition is not explicitly discussed."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Search time budgets reported: 2 hours for SafeQA, 3 hours for NaviQA. Section III.G: 'more than 234,000 tests (1,000 tests per run), for a total execution time of more than 24 days.' Table I includes per-LLM inference time for judges (Table III)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total execution time stated as 'more than 24 days' for 234,000+ tests. Azure used for cloud models, Ollama/HuggingFace for local models. Per-run budget of 2-3 hours with 6 repetitions per configuration."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results reported across 6 independent runs per configuration with standard deviation shown in Figures 2 and 3. Box plots show run-to-run variability. This captures sensitivity to stochastic initialization of the evolutionary algorithm."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: 'Results averaged over 6 runs' in Figures 2 and 3 captions. Diversity analysis repeated 10 times with averaged coverage results."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Parameters described as 'informed by preliminary experiments' (population size, search time) and 'default settings introduced by Abdessalem et al.' (mutation/crossover). No reporting of how many configurations were tried or total compute spent on tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Configuration selected via unspecified 'preliminary experiments' and defaults from prior work. No description of how many configurations were evaluated or what criterion was used to select the final parameters."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Mann-Whitney U tests performed across multiple baselines (3-4), multiple LLMs (3-6), and multiple case studies (2-3), resulting in many comparisons. No mention of Bonferroni, Holm, or other family-wise error rate corrections."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors developed STELLAR and compare it against their own implementations of baselines (RS, T-WISE) and ASTRAL. No discussion of author-evaluation bias or whether baseline implementations are fair."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 2 and 3 (bottom) show failure ratio as a function of time for all approaches, directly comparing performance at matched compute budgets. All approaches run under the same time budget (2h/3h)."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether failure count is the right measure for testing effectiveness, or whether the LLM judge (F1=0.71-0.79) introduces systematic measurement error that could differentially affect approaches."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The comparison is between testing approaches (STELLAR vs. baselines), not between models in different scaffolds. The testing framework IS the thing being tested. No scaffolding confound applies."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While STELLAR generates new test inputs dynamically, the RAG examples drawn from BeaverTails [16] (2023) and MultiWOZ [15] (2020) could have been in the training data of models tested. This temporal overlap is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the feature space design or RAG examples leak information about expected failure modes. The 5 malicious examples in SafeQA prompts could bias the types of unsafe outputs generated."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between generated test inputs across runs or between the judge training/evaluation data. RAG examples may create systematic dependencies in generated inputs."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method applied. The paper acknowledges contamination risk in static benchmarks (Section I) but does not apply any detection method to their own evaluation."
       }
     }
   },
   "claims": [
     {
       "claim": "STELLAR exposes up to 4.3× (average 2.5×) more failures than existing baseline approaches across both SafeQA and NaviQA case studies.",
       "evidence": "Figures 2 and 3 show failure counts and failure ratios across 6 LLMs and 3 AUTs. Mann-Whitney U test (α=0.05) with Vargha-Delaney A12 confirms statistical significance with large effect sizes (Section IV.F.2).",
       "supported": "strong"
     },
     {
       "claim": "GPT-4O-MINI demonstrates the best balance between performance and cost efficiency as an LLM judge, achieving F1-scores of 0.79 (SafeQA) and 0.71 (NaviQA).",
       "evidence": "Table III reports F1 scores and inference times across 7 models for both binary and continuous judgment tasks, averaged over 5 runs (Section IV.F.1).",
       "supported": "moderate"
     },
     {
       "claim": "STELLAR achieves higher failure diversity than ASTRAL, reaching up to 98% cluster coverage on GPT-5-CHAT.",
       "evidence": "Table IV shows cluster coverage for SafeQA and NaviQA. Coverage differences between STELLAR and ASTRAL are statistically significant with large effect sizes (Section IV.F.3).",
       "supported": "moderate"
     },
     {
       "claim": "STELLAR uncovered two novel failure types (F3: Name Misinterpretation, F5: Technical Output) in NaviQA-II that had not been detected through prior testing.",
       "evidence": "Section VI: domain expert with 7 years experience confirmed F3 and F5 were previously undetected. However, this is based on a single expert's assessment.",
       "supported": "weak"
     },
     {
       "claim": "Smaller local LLMs (MISTRAL 7B, DEEPSEEK-V2-16B) exhibit substantially more failures than large cloud-based models (GPT-5-CHAT).",
       "evidence": "Figures 2 and 3 show consistent pattern: MISTRAL 7B has highest failure counts (~1000 in SafeQA), GPT-5-CHAT lowest (~200). This holds across all testing approaches (Section IV.F.2).",
       "supported": "strong"
     },
     {
       "claim": "The majority of identified failures in NaviQA-II are of high severity.",
       "evidence": "Table V classifies 5 of 9 failure types as High criticality based on domain expert assessment (Section VI, Q3). However, this is a single expert's judgment.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "STELLAR, a search-based testing framework using evolutionary optimization, consistently finds 2.5× more failures than random, combinatorial, and coverage-based baselines when testing LLM applications. The framework was evaluated on safety testing of 6 LLMs (SafeQA) and navigational QA testing of an open-source and industrial BMW system (NaviQA). Results show statistical significance with large effect sizes across all comparisons. Qualitative evaluation on the industrial system uncovered 9 distinct failure types, including two previously undetected categories, with the majority classified as high severity.",
   "red_flags": [
     {
       "flag": "Conflict of interest",
       "detail": "Three of four authors are BMW employees. BMW funds the research and provides the industrial system under test (NaviQA-II). The qualitative evaluation is validated by a single BMW domain expert. This creates a dual incentive: demonstrating the tool works well AND showing the BMW system has findable flaws (justifying testing investment)."
     },
     {
       "flag": "Single-expert qualitative validation",
       "detail": "The qualitative evaluation of failure types (Section VI) and severity assessment relies entirely on one domain expert with 7 years experience. No inter-rater reliability for the failure type classification. Novel failure types F3 and F5 validated by this single expert."
     },
     {
       "flag": "No ablation study",
       "detail": "STELLAR has multiple components (evolutionary search, crossover/mutation, RAG-based generation, duplicate elimination, feature constraints) but no ablation study isolates the contribution of each. It's unclear which component drives the improvement over baselines."
     },
     {
       "flag": "Model versions unspecified",
       "detail": "All LLMs identified by marketing names (GPT-4O, GPT-5-CHAT, DEEPSEEK-V3, etc.) without API version IDs or snapshot dates. Model behavior changes across versions, making results non-reproducible."
     },
     {
       "flag": "Small human evaluation sample",
       "detail": "LLM judge validation uses only 10 BMW participants rating 30 items. No demographics, no recruitment method, no ethics approval. The judge achieves only 0.71 F1 on NaviQA, meaning ~29% of judgments may be incorrect, which could systematically bias the comparison between approaches."
     }
   ],
   "cited_papers": [
     {
       "title": "Astral: A tool for the automated safety testing of large language models",
       "authors": ["M. Ugarte", "P. Valle", "J. A. Parejo", "S. Segura", "A. Arrieta"],
       "year": 2025,
       "relevance": "State-of-the-art automated LLM testing baseline using coverage-based feature combination; directly compared against STELLAR."
     },
     {
       "title": "Jailbreaking leading safety-aligned llms with simple adaptive attacks",
       "authors": ["M. Andriushchenko", "F. Croce", "N. Flammarion"],
       "year": 2025,
       "relevance": "Demonstrates LLM jailbreaking via adaptive suffix modification, achieving up to 100% attack rates on safety-aligned LLMs."
     },
     {
       "title": "METAL: Metamorphic Testing Framework for Analyzing Large-Language Model Qualities",
       "authors": ["S. Hyun", "M. Guo", "M. A. Babar"],
       "year": 2024,
       "relevance": "Metamorphic testing framework for LLM robustness and fairness; provides character-level perturbation features used in STELLAR."
     },
     {
       "title": "Beavertails: Towards improved safety alignment of llm via a human-preference dataset",
       "authors": ["J. Ji", "M. Liu", "J. Dai"],
       "year": 2023,
       "relevance": "Safety alignment dataset used for RAG examples and judge evaluation in STELLAR's SafeQA case study."
     },
     {
       "title": "SORRY-bench: Systematically evaluating large language model safety refusal",
       "authors": ["T. Xie", "X. Qi", "Y. Zeng"],
       "year": 2025,
       "relevance": "Safety benchmark providing 13 content categories used as features in STELLAR's safety testing case study."
     },
     {
       "title": "Addressing Data Leakage in HumanEval Using Combinatorial Test Design",
       "authors": ["J. S. Bradbury", "R. More"],
       "year": 2025,
       "relevance": "Addresses data contamination in LLM benchmarks, a key motivation for dynamic test generation approaches like STELLAR."
     },
     {
       "title": "How toxic can you get? search-based toxicity testing for large language models",
       "authors": ["S. Corbo", "L. Bancale", "V. D. Gennaro"],
       "year": 2025,
       "relevance": "Search-based approach for LLM toxicity testing; closely related work in evolutionary LLM testing."
     },
     {
       "title": "Mortar: Metamorphic multi-turn testing for llm-based dialogue systems",
       "authors": ["G. Guo", "A. Aleti", "N. Neelofar", "C. Tantithamthavorn"],
       "year": 2024,
       "relevance": "Framework for testing LLM dialogue systems via metamorphic relations at the turn level; related automated testing approach."
     },
     {
       "title": "A survey on code generation with llm-based agents",
       "authors": ["Y. Dong", "X. Jiang", "J. Qian"],
       "year": 2025,
       "relevance": "Survey of LLM-based code generation agents, relevant to the broader context of LLM application testing."
     },
     {
       "title": "From llms to llm-based agents for software engineering: A survey of current, challenges and future",
       "authors": ["H. Jin", "L. Huang", "H. Cai"],
       "year": 2025,
       "relevance": "Survey of LLM-based agents for software engineering, providing context for why systematic testing of LLM applications is needed."
     },
     {
       "title": "LLMs in software security: A survey of vulnerability detection techniques and insights",
       "authors": ["Z. Sheng", "Z. Chen", "S. Gu"],
       "year": 2025,
       "relevance": "Survey on LLM vulnerability detection relevant to security testing of LLM applications."
     }
   ]
 }