ai-research-survey

scan.json (30832B)

---

 {
   "paper": {
     "title": "PRIMG: Efficient LLM-driven Test Generation Using Mutant Prioritization",
     "authors": [
       "Mohamed Salah Bouafif",
       "Mohammad Hamdaqa",
       "Edward Zulkoski"
     ],
     "year": 2025,
     "venue": "EASE 2025 (29th International Conference on Evaluation and Assessment in Software Engineering)",
     "arxiv_id": "2505.05584",
     "doi": "10.1145/3756681.3756991"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PRIMG combines ML-based mutant prioritization with LLM-driven test generation for Solidity smart contracts. The refining module dramatically improves test correctness from ~3% (single-shot) to ~33% (5 iterations), with diminishing returns beyond 5 iterations. Prioritized mutant selection significantly outperforms random selection in mutants killed across 3 real-world projects from Code4Arena. However, the overall mutation score impact was explicitly not evaluated, undermining the paper's central abstract claim.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided in Section 6: 'All the scripts and results that have been used during our experiments can be found in this github repository https://github.com/Salah-SH/llm-mutation-testing.' Also mentioned in Section 4.1.3."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 states 'All scripts and dataset can be found here' with link to GitHub repository. The projects are from publicly accessible Code4Arena."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'EC2 instance of type g5.xlarge with 26 GiB of GPU memory' (Section 4.1) and tools (SuMo, Hardhat, Ganache), but no requirements.txt, Dockerfile, or detailed library version specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub repository is referenced but the paper itself does not include a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports Z-test statistics and p-values but does not provide confidence intervals or error bars on the main results (Tables 2 and 3)."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "Pairwise Z-tests for proportions are used throughout: RQ1.1 compares single-shot vs refining (e.g., Allobase Z=-19.08, p<0.05), RQ1.2 compares loop sizes, and RQ2 compares prioritized vs random (e.g., Allobase Z=-15.98, p<0.05)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Raw proportions with baseline context are reported throughout. E.g., Table 2: Allobase single-shot 23/500 (4.6%) vs refining loop-5 201/500 (40.2%). Table 3 shows raw mutant kill counts for prioritized vs random sets, providing enough context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 500 tests per project, why 50 tests per random set in Table 3, or why 3 projects. No power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 shows 3 random test sets with varying results (e.g., Allobase: 200, 158, 8) but no standard deviation, IQR, or formal variance measure is reported. The spread across random sets is visible but not quantified."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Two baselines are used: (1) single-shot LLM generation (no refining) vs refining module in RQ1, and (2) random mutant selection vs prioritized selection in RQ2."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The only baselines are single-shot prompting and random mutant selection. No comparison with other LLM-based test generation approaches (e.g., ChatTester, AID which are cited in related work) or other prioritization techniques beyond random. The paper itself acknowledges random selection is the status quo, but contemporary LLM-based alternatives exist."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The two-component system is effectively ablated: RQ1 tests the generation module with/without refining (loop sizes 1, 5, 10), and RQ2 tests prioritized vs random selection, isolating each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two distinct metrics are used: (1) proportion of correct tests generated (syntactic + behavioral correctness) in RQ1, and (2) number of mutants killed in RQ2."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated: compilation check (syntax verification) and test execution against the PUT (behavior verification). No human evaluation of test quality, readability, or meaningfulness."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For the ML model, Section 3.2.3 describes 'Project-Only: training on mutants in all but one held-out smart contract in a single project and testing on mutants in the held-out smart contract.' The evaluation uses this hold-one-out scheme."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per project (Allobase, Particle, Quadrata) in Tables 2 and 3, with separate Z-test results for each. Table 1 provides project characteristics."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 discusses why LLMs fail to generate correct tests even after multiple iterations: hallucination, token-by-token generation inconsistencies, limited context windows, and struggles with complex multi-line fixes."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Finding 2 reports that increasing from 5 to 10 refining loops shows no significant improvement (diminishing returns). The paper also acknowledges the LLM still fails on many tasks regardless of iteration count."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims PRIMG 'significantly reduces test suite size while maintaining high mutation coverage.' However, Section 7 explicitly admits: 'this study did not evaluate the number of newly killed mutants or the impact of the new tests on the overall mutation score.' The central claim about maintaining mutation coverage is unsupported by the paper's own evidence."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims ('refining module significantly improves correctness,' 'prioritization outperforms random') are supported by controlled comparisons: with/without refining (single-shot vs iterative), and prioritized vs random selection, both with Z-test confirmation. The ablation-like design adequately supports these causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Efficient LLM-driven Test Generation Using Mutant Prioritization' is unbounded — no mention of Solidity. The conclusion claims 'its design and evaluation methodology are fundamentally adaptable to various programming languages and models' despite testing only on 3 Solidity projects with a single LLM (Llama 3.1 8B)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses validity threats but not alternative explanations for the observed results. No consideration of whether prioritized mutants are simply easier to kill, whether the LLM has seen Code4Arena projects in training, or whether the refining improvement is due to random restarts vs actual error-driven refinement."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test correctness (compilation + passing PUT) and mutants killed, and frames these directly as test generation effectiveness. There is no significant gap between what is measured and what is claimed at the measurement level, though the abstract overclaims about mutation coverage."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 states 'We use llama 3.1 with 8B parameter.' Llama 3.1 8B identifies the model family, version, and size. For the ML model: 'ridge regression' with specific configurations. However, the Llama variant (base vs instruct) is not specified."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1.1 describes prompt components (PUT, mutant code, initial test file) and Section 3.1.2 describes augmented prompts conceptually, but the actual prompt text is never provided. Only the structure is described: 'CONCAT(initTest, prioritizedMutant, PUT).'"
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "ML model hyperparameters are reported (ridge regression, random forest with max depth 3 and 10 trees). However, LLM inference parameters (temperature, top-p, max tokens, sampling strategy) are completely absent, which significantly affects generation behavior."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The workflow is described in detail: Algorithm 1 (PRIMG main flow), Algorithm 2 (RefiningTest with syntax/behavior verification loop), Figure 1 (system overview), and the prioritization module (Section 3.2). The retry logic, feedback mechanism (error logs fed back as augmented prompts), and loop termination conditions are specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "Project selection criteria are stated (>10 smart contracts, initial test suite, mutation score <80%) but no pipeline counts are given — how many Code4Arena projects were considered initially, how many passed each criterion. The mutant generation process is described but intermediate counts (total mutants, killed vs survived) are not reported."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threat to Validity' provides substantive discussion across multiple paragraphs covering prompt design, mutation operators, testing framework, missing metrics, and resource constraints."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses threats specific to this study: (1) SuMo mutation operators may not cover all smart contract vulnerabilities, (2) exclusive use of Hardhat may limit applicability, (3) did not evaluate mutation score impact, (4) limited set of prompt techniques explored, (5) resource constraints restricted scope."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 7 mentions framework-specific and operator-specific limitations, the conclusion actively extends beyond scope: 'its design and evaluation methodology are fundamentally adaptable to various programming languages and models.' The paper does not explicitly state what the results do NOT show beyond generic validity threats."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 states 'All the scripts and results that have been used during our experiments can be found in this github repository https://github.com/Salah-SH/llm-mutation-testing.'"
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.3 describes using Code4Arena as the source, the selection criteria for projects (number of contracts, initial test suite, mutation score), and Table 1 provides project characteristics. SuMo is described as the mutant generator with 25 Solidity-specific and 19 general-purpose operators."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from Code4Arena, a public platform for smart contract auditing. Project selection criteria are described in Section 4.1.3."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The overall flow is described conceptually (project selection → mutation testing → ML labeling → test generation) but counts at intermediate stages are missing. Total mutants generated per project, killed vs survived breakdown, and feature extraction statistics are not reported."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. No mention of grants, sponsors, or funding agencies."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: two authors from Polytechnique Montréal, one (Edward Zulkoski) from Quantstamp, Inc., a smart contract security auditing company."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Notably, one author is from Quantstamp Inc., a company that performs smart contract security audits and would directly benefit from better automated testing tools. This conflict is not acknowledged."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement. Edward Zulkoski's affiliation with Quantstamp (a commercial smart contract auditing company) represents a potential financial interest that is not declared beyond the affiliation line."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Llama 3.1 8B to generate test cases for publicly available Code4Arena projects but does not state the model's training data cutoff date. The LLM may have seen these projects during training."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Code4Arena projects appeared in Llama 3.1's training data. Since Code4Arena projects are publicly available on GitHub, this is a real contamination risk."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Code4Arena projects and their test suites are publicly available online. Llama 3.1 was trained on web data that likely includes GitHub repositories. This contamination risk is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The evaluation is entirely automated."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated test generation on smart contract code."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, tokens consumed, or wall-clock time reported. The approach calls Llama 3.1 up to 10 times per mutant across 500 mutants per project, but the cost of this is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.1 mentions 'EC2 instance of type g5.xlarge with 26 GiB of GPU memory' but does not quantify total GPU hours, experiment duration, or overall compute budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. The LLM generation is inherently stochastic but no analysis of variability across seeds or random initializations is provided. Table 3 shows variation across random mutant selections but not across repeated runs of the same configuration."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of test generation attempts is stated: 500 per project (Section 4.2.2). Loop sizes of 1, 5, and 10 are tested (Section 4.2.1). Table 3 uses 3 random sets of 50 tests and 1 prioritized set."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.3 describes the ML model search: 'We evaluated every combination of the following choices' — 2 models (ridge regression, random forest) × 1 feature set × 2 training settings. The search space and selection method are documented, though compute cost is not."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2.3 states: 'Building on findings in [15], we choose the following configuration: Model=Ridge regression, Features=All, Training Setting=Project-Only.' Configuration selected based on prior work and own evaluation across combinations."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple Z-tests are performed across 3 projects and multiple comparisons (1 vs 5, 1 vs 10, 5 vs 10 trials; prioritized vs random). No Bonferroni or other multiple comparison correction is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own PRIMG system against a random baseline they also implement. No acknowledgment of self-comparison bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The loop size analysis (1, 5, 10 iterations) implicitly relates compute to performance, but no explicit compute-vs-performance analysis is provided. The compute cost of prioritized vs random selection is not compared."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Code4Arena projects are representative of smart contracts generally, whether mutation score is a valid proxy for test quality, or whether the 3 selected projects are sufficient to support general claims."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The prioritized vs random comparison in RQ2 uses the same test generation scaffold (same LLM, same refining process), controlling for the scaffold confound. The difference is solely in mutant selection strategy."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Code4Arena projects existed before Llama 3.1's training cutoff. These projects are publicly available and could have been in the training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompts (which include the full PUT and mutant code) provide information that leaks the correct answer. No analysis of whether the LLM is generating tests from memorization vs understanding."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the 3 projects or whether patterns learned from one Code4Arena project transfer to others due to structural similarity."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference, or decontamination analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "The refining module significantly improves correctness of LLM-generated tests compared to single-shot generation.",
       "evidence": "Table 2 shows Allobase: 4.6% (single-shot) → 40.2% (5 iterations); Particle: 2% → 30.2%; Quadrata: 4.2% → 28%. Z-tests confirm significance (e.g., Allobase Z=-19.08, p<0.05). Section 4.2.1.",
       "supported": "strong"
     },
     {
       "claim": "No significant improvement in test correctness when increasing refining loops from 5 to 10 iterations.",
       "evidence": "Z-tests show no significant difference between 5 and 10 trials across all projects: Allobase Z=-0.22, Particle Z=0.00, Quadrata Z=-0.35, all p>0.05. Section 4.2.1, Table 2.",
       "supported": "moderate"
     },
     {
       "claim": "Prioritized mutant selection significantly outperforms random selection in killing mutants.",
       "evidence": "Table 3 shows prioritized tests kill more mutants than random sets across all projects. Z-tests: Allobase Z=-15.98, Particle Z=-12.94, Quadrata Z=-14.12, all p<0.05. Section 4.2.2.",
       "supported": "moderate"
     },
     {
       "claim": "PRIMG significantly reduces test suite size while maintaining high mutation coverage.",
       "evidence": "Abstract claim. Section 7 explicitly admits: 'this study did not evaluate the number of newly killed mutants or the impact of the new tests on the overall mutation score.' The mutation coverage claim is unsupported.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Tiny sample size",
       "detail": "Only 3 Solidity projects from Code4Arena are evaluated. This is insufficient to support the generalization claims made in the title and conclusion."
     },
     {
       "flag": "Missing key metric",
       "detail": "Section 7 explicitly admits the paper 'did not evaluate the number of newly killed mutants or the impact of the new tests on the overall mutation score,' despite mutation coverage being a central claim in the abstract."
     },
     {
       "flag": "Abstract overclaims",
       "detail": "The abstract claims 'maintaining high mutation coverage' but the paper does not measure mutation score improvement. This is a direct contradiction between the abstract and the evaluation."
     },
     {
       "flag": "No comparison with contemporary approaches",
       "detail": "Despite citing ChatTester and AID in related work, PRIMG is only compared against random mutant selection. No head-to-head comparison with other LLM-based test generation tools."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Co-author Edward Zulkoski is from Quantstamp Inc., a commercial smart contract security auditing company that would directly benefit from better automated testing tools. No COI disclosure or funding statement is present."
     },
     {
       "flag": "Incomplete manuscript",
       "detail": "Section 4.1.1 contains an incomplete sentence: 'Llama3.1 is the leading' — the sentence is cut off, suggesting the manuscript was not fully proofread."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "Code4Arena projects are publicly available on GitHub. Llama 3.1 was trained on web-scale data likely including these repositories. The LLM may have memorized test patterns for these specific contracts, inflating correctness rates."
     }
   ],
   "cited_papers": [
     {
       "title": "Effective test generation using pre-trained large language models and mutation testing",
       "authors": ["Arghavan Moradi Dakhel", "Amin Nikanjam", "Vahid Majdinasab", "Foutse Khomh", "Michel C Desmarais"],
       "year": 2024,
       "relevance": "Directly related: uses mutation testing to guide LLMs in test generation, the primary inspiration for PRIMG."
     },
     {
       "title": "No more manual tests? evaluating and improving chatgpt for unit test generation",
       "authors": ["Zhiqiang Yuan", "Yiling Lou", "Mingwei Liu"],
       "year": 2023,
       "arxiv_id": "2305.04207",
       "relevance": "ChatTester: iteratively refines LLM-generated test cases, directly comparable approach to PRIMG's refining module."
     },
     {
       "title": "LLM-Powered Test Case Generation for Detecting Tricky Bugs",
       "authors": ["Kaibo Liu", "Yiyang Liu", "Zhenpeng Chen"],
       "year": 2024,
       "arxiv_id": "2404.10304",
       "relevance": "AID framework combining LLMs with differential testing for fault-revealing test generation."
     },
     {
       "title": "VALTEST: Automated Validation of Language Model Generated Test Cases",
       "authors": ["Hamed Taherkhani", "Hadi Hemmati"],
       "year": 2024,
       "arxiv_id": "2411.08254",
       "relevance": "Addresses accuracy issues in LLM-generated tests, directly relevant to test validation challenges."
     },
     {
       "title": "Large-scale, Independent and Comprehensive study of the power of LLMs for test case generation",
       "authors": ["Wendkûuni C Ouédraogo", "Kader Kaboré", "Haoye Tian"],
       "year": 2024,
       "arxiv_id": "2407.00225",
       "relevance": "Large-scale evaluation of LLM capabilities for test generation across multiple settings."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper — foundational work on evaluating LLMs for code generation."
     },
     {
       "title": "Large language models for software engineering: A systematic literature review",
       "authors": ["Xinyi Hou", "Yanjie Zhao", "Yue Liu"],
       "year": 2023,
       "relevance": "Comprehensive survey of LLMs for SE tasks including test generation."
     },
     {
       "title": "What's Wrong with Your Code Generated by Large Language Models? An Extensive Study",
       "authors": ["Shihan Dou", "Haoxiang Jia", "Shenxi Wu"],
       "year": 2024,
       "arxiv_id": "2407.06153",
       "relevance": "Studies quality issues in LLM-generated code including functional bugs and hallucinations."
     },
     {
       "title": "LLMorpheus: Mutation Testing using Large Language Models",
       "authors": ["Frank Tip", "Jonathan Bell", "Max Schäfer"],
       "year": 2024,
       "arxiv_id": "2404.09952",
       "relevance": "Uses LLMs for mutation generation itself, complementary to PRIMG's use of LLMs for test generation."
     },
     {
       "title": "A deep dive into large language models for automated bug localization and repair",
       "authors": ["Soneya Binta Hossain", "Nan Jiang", "Qiang Zhou"],
       "year": 2024,
       "relevance": "Studies LLM capabilities and limitations for automated bug detection and repair in software."
     },
     {
       "title": "Leveraging Propagated Infection to Crossfire Mutants",
       "authors": ["Hang Du", "Vijay Krishna Palepu", "James A Jones"],
       "year": 2024,
       "arxiv_id": "2411.09846",
       "relevance": "Advances mutation testing methodology by leveraging propagated infection for mutant detection."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Solidity smart contract developers could use PRIMG's approach for automated test generation, though the tool is domain-specific and the LLM choice (Llama 3.1 8B) is modest."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Results confirm expectations: iterative refinement helps LLM output, and informed selection outperforms random selection."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Smart contract testing is security-relevant, but this paper improves testing rather than demonstrating attacks or novel vulnerabilities."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or challenges to established beliefs."
     },
     "demo_ability": {
       "score": 2,
       "justification": "GitHub repository with scripts and datasets is provided, making the tool somewhat reproducible for those with the infrastructure."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Academic group from Polytechnique Montréal and small company Quantstamp; not a major AI lab."
     }
   }
 }