ai-research-survey

scan.json (30080B)

---

 {
   "paper": {
     "title": "The Prompt Alchemist: Automated LLM-Tailored Prompt Optimization for Test Case Generation",
     "authors": [
       "Shuzheng Gao",
       "Chaozheng Wang",
       "Cuiyun Gao",
       "Xiaoqian Jiao",
       "Chun Yong Chong",
       "Shan Gao",
       "Michael R. Lyu"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2501.01329",
     "doi": "10.48550/arXiv.2501.01329"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "MAPS, an automated prompt optimization method for LLM-based test case generation, outperforms four state-of-the-art prompt optimization baselines by 6.19% line coverage and 5.03% branch coverage on the Defects4J benchmark across ChatGPT, Llama-3.1, and Qwen2. The method demonstrates that different LLMs benefit from different optimized prompts, with each model achieving best performance on its own tailored prompt. An ablation study shows all three modules (diversity-guided prompt generation, failure-driven rule induction, domain contextual knowledge extraction) contribute meaningfully, with contextual knowledge extraction providing the largest individual gain.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A replication package is provided on Zenodo at https://zenodo.org/records/14287744, referenced in Section VIII and throughout the paper (reference [29])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses the publicly available Defects4J benchmark. The authors state 'We present the sampled development set Ddev in our replication package' (Section IV-D) along with seed prompts and prompt templates."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only hardware is mentioned: 'Ubuntu 20.04 server with a 112-core Intel Xeon Platinum CPU' (Section IV-D). No software environment specifications (requirements.txt, dependency versions, Python version, etc.) are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references a replication package on Zenodo but does not provide step-by-step reproduction instructions within the paper itself. No README with commands, no 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported in parentheses for MAPS results across all tables (e.g., '53.80 (0.04)' in Table IV). This provides uncertainty information."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests (p-values, t-tests, etc.) are reported anywhere in the paper. Claims like 'outperforms baseline methods by a large margin' are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported as absolute percentage differences with baseline context throughout (e.g., '6.19% higher line coverage rate' over the strongest baseline, with both baseline and MAPS scores visible in Tables IV-VI)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 10 bugs were chosen as the development set, or why 147 bugs from 5 projects constitute a sufficient test set. No power analysis is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "'We repeat MAPS three times and report its average results and variance to eliminate the influence of sampling and fluctuations in LLM' (Section IV-D). Standard deviations are shown in Tables IV-VI."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four prompt optimization baselines are compared: APE, OPRO, EVOPROMPT (GA), and EVOPROMPT (DE), plus a Basic prompt baseline (Section IV-C)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include APE (ICLR 2023), OPRO (ICLR 2024), and EVOPROMPT (ICLR 2024), all contemporary prompt optimization methods at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table VIII presents a comprehensive ablation study removing each of the three modules individually and testing only the context extraction module, across all three LLMs (RQ3, Section V-C)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two evaluation metrics are used: line coverage and branch coverage, reported for all experiments (Section IV-B)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using coverage metrics. No human evaluation of test case quality, readability, or usefulness is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The test set includes the development set: 'we randomly sample ten bugs from the Defects4J benchmark as our development set Ddev and use all bugs as test set Dtest' (Section IV-D). The 10 dev bugs used for prompt optimization are included in the 147 test bugs."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Per-project breakdowns are provided for all five Defects4J projects (Chart, Cli, Csv, Gson, Lang) in Tables IV-VI, showing meaningful performance variation across projects."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-A presents two case studies showing how MAPS addresses specific failure patterns (exception handling in Listing 1, abstract class instantiation in Listing 2). Motivating examples in Section II-B also discuss failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Baseline methods APE and OPRO sometimes perform worse than the basic prompt (Table IV: APE achieves 44.58 vs Basic 45.56 line coverage). The ablation study shows which removals hurt most. RQ4 shows diminishing returns with some parameter changes."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '6.19% higher line coverage rate and a 5.03% higher branch coverage rate on average' compared to the strongest baseline. Tables IV-VI show MAPS achieving 53.80, 50.59, 45.51 line coverage vs EVOPROMPT(GA) at 46.63, 46.52, 38.17, yielding improvements of 7.17, 4.07, 7.34 averaging ~6.19%."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about module contributions ('removing X reduces performance by Y%') are supported by a controlled ablation study (Table VIII) where single components are removed while holding others constant. This is adequate single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automated LLM-Tailored Prompt Optimization for Test Case Generation' broadly, but experiments are limited to Java (Defects4J). The paper claims 'our method is language-agnostic and can be easily adapted to other programming languages' (Section VI-C) without evidence. The claim of language-agnostic generalizability is unbounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section VI-C discusses 'Limited LLMs' and 'Limited Programming Languages' as threats to validity, but these are scope limitations, not alternative explanations for the observed results. No confounds are discussed (e.g., whether the improvement comes from the additional context vs. the optimization, or whether the dev/test overlap biases results)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures line coverage and branch coverage and frames results in terms of these same metrics without inflating to broader claims about test quality or software reliability. The measurements match the granularity of claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are provided: 'gpt-3.5-turbo-0125', 'Llama-3.1-70B-Instruct', and 'Qwen2-72B-Instruct' (Section IV-C)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Prompt templates for modification, reflection, and transformation are shown in Fig. 3. The paper states 'The seed prompts, and all prompt templates used in our work can be found in our replication package' (Section IV-D) with a Zenodo URL. Example final prompts are shown in Fig. 6."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "MAPS-specific parameters are reported (seed prompts=5, N=2, I=5), but LLM inference hyperparameters (temperature, top-p, max tokens) are not reported for any of the three models used. These significantly affect LLM output."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The iterative optimization pipeline is described in detail with Algorithms 1-2, Fig. 1 workflow diagram, and separate subsections for each module (Sections III-B through III-D). The three-module architecture with failure feedback loops constitutes scaffolding and is well documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents how focal methods are extracted from Defects4J projects, how in-file and cross-file context is gathered (Section III-B), how seed prompts are obtained, and how the development set is sampled (Section IV-D). Table III provides dataset statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VI-C 'Threats to Validity' provides a dedicated subsection discussing two main threats."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'some models are not covered' (limited LLMs) and 'Defects4J benchmark, which only contains Java projects' (limited programming languages). These identify concrete gaps in the evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While the limitations section mentions Java-only and three LLMs, it then claims 'MAPS is model-agnostic' and 'language-agnostic and can be easily adapted to other programming languages' without evidence. The paper does not clearly state what the results do NOT show — it frames limitations as future work rather than boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "A replication package is on Zenodo, but the paper does not confirm that raw per-bug coverage results are included. Only aggregate results (project-level averages) are shown in the paper."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: Defects4J benchmark with five Java projects, specific bug versions listed in Table III, focal method extraction procedure described in Section III-B."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is the standard Defects4J benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from benchmark selection → focal method extraction → context extraction → prompt optimization → test generation → coverage evaluation is documented across Sections III and IV. Table III shows dataset statistics at each level (bugs, focal classes, focal methods)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is visible in the paper text. Funding is not disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: The Chinese University of Hong Kong, Harbin Institute of Technology (Shenzhen), Monash University Malaysia, and an independent researcher."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, making it impossible to assess funder independence. The absence of funding disclosure is a transparency gap."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three models (gpt-3.5-turbo-0125, Llama-3.1-70B-Instruct, Qwen2-72B-Instruct) used in experiments."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J is a widely-used benchmark published in 2014 and extensively discussed online. No analysis of whether test examples appear in any model's training data is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J has been publicly available since 2014 and is a standard benchmark in software engineering research. All three models were trained well after its publication. The contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is entirely automated benchmark-based."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, token counts, or wall-clock times are reported. MAPS requires multiple iterative LLM calls for optimization (5 iterations × multiple prompt evaluations) but the cost is never quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Only hardware is mentioned ('Ubuntu 20.04 server with a 112-core Intel Xeon Platinum CPU'). No total compute budget (GPU hours, API spend, total wall-clock time) is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "'We repeat MAPS three times and report its average results and variance' (Section IV-D). Standard deviations across runs are shown in Tables IV-VI."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states 'We repeat MAPS three times' (Section IV-D). However, baseline methods appear to be run only once."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "RQ4 (Section V-D) studies parameter sensitivity by varying seed prompts (3-6), N (1-3), and iterations (1-3), but the total computational budget for this search is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section V-D provides justification for parameter choices: 'the improvements over five seed prompts are marginal. Therefore, we set the number of seed prompts to five.' Similar justification for N=2 and I=5."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across 3 LLMs × 5 projects × 2 metrics, but no significance tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baseline methods and compare against their own system. No discussion of author-evaluation bias or whether baselines were given comparable tuning effort."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "MAPS requires iterative optimization with multiple LLM calls per iteration (prompt generation, evaluation, reflection, rule induction) while baselines use simpler optimization. This compute difference is not quantified or discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Line coverage and branch coverage are used as sole quality measures without discussing whether coverage actually captures test effectiveness, fault detection, or practical test utility. No discussion of coverage metric limitations."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All methods are compared using the same underlying LLMs. The ablation study (Table VIII) separates the contributions of MAPS's three modules, addressing what comes from the optimization scaffold vs. the context extraction."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J bugs date from 2006-2016. All models (GPT-3.5, Llama-3.1, Qwen2) were trained well after these dates. The models may have seen Defects4J bugs and their fixes during training. Not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MAPS provides cross-file context (subclass information, constructor signatures) as features. Whether this constitutes information that would be available in a realistic usage scenario is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The development set (10 bugs) is a subset of the test set (all 147 bugs). This overlap between optimization and evaluation data is not discussed as a leakage concern."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are employed (no canary strings, no membership inference, no temporal splits, no decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "MAPS outperforms baseline prompt optimization methods by 6.19% higher line coverage and 5.03% higher branch coverage on average across three LLMs.",
       "evidence": "Tables IV-VI show MAPS achieving average line coverage of 53.80/50.59/45.51 vs strongest baselines at 46.63/46.52/39.41 across ChatGPT/Llama-3.1/Qwen2 on Defects4J (Section V-A).",
       "supported": "moderate"
     },
     {
       "claim": "MAPS effectively generates LLM-tailored prompts, with each model achieving best performance on its own optimized prompt.",
       "evidence": "Table VII shows each model achieves highest line coverage with its own prompt: ChatGPT 53.80% vs 51.35%/51.14%, Llama-3.1 50.59% vs 41.92%/43.98%, Qwen2 45.51% vs 35.98%/44.94% (Section V-B).",
       "supported": "moderate"
     },
     {
       "claim": "All three modules (diversity-guided prompt generation, failure-driven rule induction, domain contextual knowledge extraction) contribute to MAPS performance.",
       "evidence": "Table VIII ablation study shows removing any module degrades performance across all three LLMs. Removing contextual knowledge causes the largest drop (e.g., -9.64% line coverage on ChatGPT) (Section V-C).",
       "supported": "strong"
     },
     {
       "claim": "MAPS generates more diverse prompts than baseline methods during optimization.",
       "evidence": "Section VI-A reports average edit distance of 27.0 for MAPS vs 9.3 for OPRO, but this is measured only against OPRO, not all baselines, and on a single metric.",
       "supported": "weak"
     },
     {
       "claim": "Existing prompt optimization methods struggle to produce effective prompts for test case generation.",
       "evidence": "Tables IV-VI show APE and OPRO sometimes performing worse than the basic prompt (e.g., APE achieves 44.58% vs Basic 45.56% line coverage on ChatGPT), with best baseline improving only 1.07% over basic (Section V-A).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Dev/test overlap",
       "detail": "The development set (10 bugs used for prompt optimization) is a subset of the test set (all 147 bugs). This means prompts were optimized partially on data included in the final evaluation, inflating reported performance."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All claims of outperformance are based on comparing point estimates without any statistical testing (no p-values, t-tests, or confidence intervals on differences). With only 3 runs of MAPS, the observed differences may not be statistically significant."
     },
     {
       "flag": "Unaddressed benchmark contamination",
       "detail": "Defects4J has been publicly available since 2014 and is one of the most widely-used Java benchmarks. All three evaluated models were trained well after its publication, creating substantial contamination risk that is never discussed."
     },
     {
       "flag": "No cost comparison with baselines",
       "detail": "MAPS requires multiple iterative LLM calls (prompt generation, evaluation, reflection, rule induction, rule validation) across 5 iterations. The computational cost is never reported or compared to simpler baselines, making practical applicability unclear."
     },
     {
       "flag": "Baselines not given equal treatment",
       "detail": "MAPS is run 3 times with variance reported, but baseline methods appear to be run only once (no standard deviations shown for baselines in Tables IV-VI). This asymmetry makes comparisons less rigorous."
     },
     {
       "flag": "Small development set",
       "detail": "Only 10 bugs (out of 147) are used as the development set for prompt optimization. The representativeness of this small sample for optimizing prompts is not justified or analyzed."
     }
   ],
   "cited_papers": [
     {
       "title": "Unit test case generation with transformers and focal context",
       "authors": ["M. Tufano", "D. Drain", "A. Svyatkovskiy", "S. K. Deng", "N. Sundaresan"],
       "year": 2020,
       "arxiv_id": "2009.05617",
       "relevance": "Pioneering deep learning approach for test case generation using transformers, directly relevant to LLM-based code generation evaluation."
     },
     {
       "title": "A3test: Assertion-augmented automated test case generation",
       "authors": ["S. Alagarsamy", "C. Tantithamthavorn", "A. Aleti"],
       "year": 2023,
       "relevance": "State-of-the-art non-LLM deep learning baseline for test case generation, used as comparison in this paper."
     },
     {
       "title": "Evaluating and improving chatgpt for unit test generation",
       "authors": ["Z. Yuan", "M. Liu", "S. Ding", "K. Wang", "Y. Chen", "X. Peng", "Y. Lou"],
       "year": 2024,
       "relevance": "Evaluates ChatGPT for test case generation with iterative refinement; directly studies LLM capability in software testing."
     },
     {
       "title": "Connecting large language models with evolutionary algorithms yields powerful prompt optimizers",
       "authors": ["Q. Guo", "R. Wang", "J. Guo", "B. Li", "K. Song", "X. Tan", "G. Liu", "J. Bian", "Y. Yang"],
       "year": 2024,
       "relevance": "State-of-the-art prompt optimization method (EVOPROMPT) using evolutionary algorithms, key baseline in this paper."
     },
     {
       "title": "Large language models are human-level prompt engineers",
       "authors": ["Y. Zhou", "A. I. Muresanu", "Z. Han", "K. Paster", "S. Pitis", "H. Chan", "J. Ba"],
       "year": 2023,
       "relevance": "Foundational work on automated prompt engineering (APE), baseline method and key related work."
     },
     {
       "title": "Large language models as optimizers",
       "authors": ["C. Yang", "X. Wang", "Y. Lu", "H. Liu", "Q. V. Le", "D. Zhou", "X. Chen"],
       "year": 2024,
       "relevance": "OPRO method using LLMs for optimization including prompt optimization, key baseline in this paper."
     },
     {
       "title": "Codamosa: Escaping coverage plateaus in test generation with pre-trained large language models",
       "authors": ["C. Lemieux", "J. P. Inala", "S. K. Lahiri", "S. Sen"],
       "year": 2023,
       "relevance": "Combines LLMs with search-based testing to improve coverage, relevant to LLM-assisted test generation."
     },
     {
       "title": "Code-aware prompting: A study of coverage-guided test generation in regression setting using LLM",
       "authors": ["G. Ryan", "S. Jain", "M. Shang", "S. Wang", "X. Ma", "M. K. Ramanathan", "B. Ray"],
       "year": 2024,
       "relevance": "Studies coverage-guided LLM prompting for test generation, directly relevant to prompt engineering for testing."
     },
     {
       "title": "Keep the conversation going: Fixing 162 out of 337 bugs for $0.42 each using chatgpt",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "CHATRepair uses iterative LLM interaction for program repair, relevant to LLM-based software engineering capabilities."
     },
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["J. Wang", "Y. Huang", "C. Chen", "Z. Liu", "S. Wang", "Q. Wang"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLM-based software testing, directly relevant to the survey scope."
     },
     {
       "title": "Search-based llms for code optimization",
       "authors": ["S. Gao", "C. Gao", "W. Gu", "M. Lyu"],
       "year": 2024,
       "relevance": "Combines search-based methods with LLMs for code intelligence, relevant to LLM optimization approaches."
     },
     {
       "title": "Self-edit: Fault-aware code editor for code generation",
       "authors": ["K. Zhang", "Z. Li", "J. Li", "G. Li", "Z. Jin"],
       "year": 2023,
       "relevance": "Uses compiler error messages to improve LLM code generation, relevant to feedback-driven LLM coding approaches."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Prompt optimization for test generation has clear practitioner value, though the iterative optimization pipeline requires significant setup and compute."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that different LLMs need different prompts is somewhat expected; the contribution is in automating the optimization rather than a surprising finding."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety, security, or risk concerns raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversial claims or conflicts; straightforward method paper."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Replication package on Zenodo is available but it is not a polished pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known models (ChatGPT, Llama) but the paper and method themselves are not from a famous AI lab."
     }
   }
 }