ai-research-survey

scan.json (30005B)

---

 {
   "paper": {
     "title": "LLM-Powered Test Case Generation for Detecting Bugs in Plausible Programs",
     "authors": [
       "Kaibo Liu",
       "Zhenpeng Chen",
       "Yiyang Liu",
       "Jie M. Zhang",
       "Mark Harman",
       "Yudong Han",
       "Yun Ma",
       "Yihong Dong",
       "Ge Li",
       "Gang Huang"
     ],
     "year": 2024,
     "venue": "arXiv preprint",
     "arxiv_id": "2404.10304",
     "doi": "10.48550/arXiv.2404.10304"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TrickCatcher, a three-stage LLM-based differential testing approach (PUT-guided variant generation, generator-based input generation, diversity-driven oracle construction), achieves F1 scores of 41.31%, 42.35%, and 51.34% on TrickyBugs (C++), TrickyBugs (Python), and EvalPlus respectively, outperforming the best baseline (Differential Prompting Plus) by 1.66× F1. The approach generates up to 16× fewer false positives than baselines on correct programs. An ablation study confirms each component contributes, and a counterintuitive finding shows that even buggy program variants (23.2% of useful variants) can produce correct test oracles.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'Code and data used are available at https://github.com/RinCloud/TrickCatcher.' A concrete repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two public datasets: TrickyBugs (MIT license) and EvalPlus (Apache 2.0), both publicly available. The GitHub repository also claims to include data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup details are provided in the paper. Only the model name (gpt-3.5-turbo-0125) and the CYaRon library are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The combinatorial repetition method is described in Appendix B but does not constitute runnable reproduction instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and all other result tables report point estimates only (e.g., 41.31% F1) with no confidence intervals, error bars, or ± notation despite claiming averaged results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TrickCatcher 'significantly outperforms' baselines (e.g., '1.66× F1') based solely on comparing numbers without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with full baseline context: '1.80×, 2.65×, and 1.66× the recall, precision, and F1 score' of DPP. Table 1 provides absolute values for both methods across all configurations."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 366 human-written and 151 AI-generated plausible programs are sufficient. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite describing a combinatorial averaging methodology in Appendix B (e.g., C(10,4)=210 rounds), the paper reports only averaged point estimates in all tables without any standard deviation, IQR, or spread measure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: DirectChat (CHAT), Differential Prompting Plus (DPP), and Automated Program Repair (APR), described in Section 5.4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Differential Prompting (Li et al., 2023) is cited as the state-of-the-art in LLM-based test case generation for bug detection. The baselines are recent and represent the current landscape."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 (Section 6.3, RQ3) presents an ablation study with 6 patterns showing contributions of each component: PUT-guided program generation, generator-based input generation, and diversity-driven differential testing."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: recall, precision, and F1 score, all reported in Table 1 and throughout the evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated via comparison with canonical program outputs and official checkers. Manual input validity verification for TrickyBugs is a setup step, not evaluation of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "TrickyBugs provides additional test cases beyond the existing suite, and EvalPlus provides extra test cases beyond base tests. These pre-defined ground-truth test sets are not used for any tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (TrickyBugs C++, TrickyBugs Python, EvalPlus), by number of variants k (2-10), and by task difficulty (RQ5, Figures 7-8)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure analysis or specific examples of where TrickCatcher fails are provided. The paper discusses aggregate false positive counts (RQ2) but does not examine individual failure cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the naive approach produces 40.10% invalid test inputs (Section 1). The ablation study (Table 2) shows that removing components degrades performance. Limitations of directly generating inputs and programs from specifications are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 1.80×, 2.65×, and 1.66× improvements in recall, precision, and F1. Table 1 supports these with specific numbers (e.g., TrickCatcher F1 41.31% vs DPP 24.95% on TrickyBugs C++)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about component contributions are supported by the ablation study (Table 2), which systematically varies components (PG, IG, DT) in a controlled manner, constituting adequate controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title frames the approach broadly ('Detecting Bugs in Plausible Programs') but it is tested only on competition-level programming problems (AtCoder-style tasks in TrickyBugs and HumanEval-level tasks in EvalPlus). No explicit acknowledgment that results may not generalize to industrial software or non-specification-driven programs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for why TrickCatcher outperforms baselines. Confounds such as the quality of prompts, model memorization of competition problems, or differences in compute cost between methods are not considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures precision, recall, and F1 for bug detection in plausible programs, which directly matches the claimed contribution. No proxy gap exists — the measurement and the framing align."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies 'gpt-3.5-turbo-0125' (Section 5.5), which includes a snapshot date identifier. DeepSeek-v3 is also named in the generalization experiment (Section 7.2)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 4 provide the actual prompt templates for program variant generation and test input generator generation, with placeholders for specification and code clearly indicated."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM API hyperparameters are reported — no temperature, top-p, max tokens, or sampling settings are mentioned for gpt-3.5-turbo-0125 or deepseek-v3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "TrickCatcher is a sequential three-step pipeline (generate variants → generate inputs → differential testing), not an agentic scaffolding system with tools, retry logic, or memory."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes how programs were selected: TrickyBugs provides 251 C++ and 115 Python plausible programs; EvalPlus programs were filtered from pre-generated LLM code samples to retain those passing base tests but failing extra tests, yielding 151 programs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 8 (Conclusion), discussing three specific limitations: budget constraints, LLM uncertainty, and data leakage risk."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: gpt-3.5-turbo was chosen due to budget constraints (not ideal), multiple repetitions were used to mitigate LLM randomness, and TrickyBugs was released after the model's training cutoff to address data leakage."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not bound claims to competition-level programming problems, does not acknowledge inapplicability to programs without clear formal specifications, and does not discuss what settings were excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The GitHub repository (https://github.com/RinCloud/TrickCatcher) claims to provide code and data. Both underlying datasets (TrickyBugs and EvalPlus) are publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes both datasets: TrickyBugs contains coding tasks from AtCoder with plausible programs from real participants; EvalPlus is a code generation benchmark with 164 tasks filtered to 151 with AI-generated plausible programs."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from standard public benchmarks (TrickyBugs and EvalPlus)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: datasets are described, filtering criteria for EvalPlus are stated (pass base tests, fail extra tests), program variant filtering via existing test suites is described, and the combinatorial repetition methodology is detailed in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists: National Key R&D Program (2023YFB4503801), NSFC (62192733, 62192730), Hubei Province (2023BAA024), and ITEA Genius/GreenCode projects via InnovateUK."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Peking University, Nanyang Technological University, King's College London, University College London."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from national science foundations and government R&D programs (NSFC, National Key R&D, InnovateUK) which have no financial stake in TrickCatcher's performance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff date for gpt-3.5-turbo-0125. The Limitations section implies temporal awareness ('TrickyBugs was released after gpt-3.5-turbo-0125') but does not state the actual cutoff date."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section addresses this: 'the TrickyBugs dataset we used was released after gpt-3.5-turbo-0125, and EvalPlus explicitly prohibits its use for training LLMs. Moreover, the poor performance of the three LLM-based baselines further suggests that data leakage is not a main concern.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section directly addresses contamination risk with two arguments: temporal ordering (TrickyBugs released after model) and licensing restriction (EvalPlus prohibits training use), plus empirical evidence (poor baseline performance suggests no memorization)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Evaluation is automated using benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or wall-clock time are reported despite the method requiring multiple LLM calls per program (generating 10 variants + input generators). The paper mentions 'budget constraints' as a reason for choosing gpt-3.5-turbo but never quantifies the actual cost."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget, GPU hours, or API spend is reported. The paper acknowledges choosing gpt-3.5-turbo for cost reasons but provides no figures."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite describing a combinatorial repetition approach in Appendix B, the paper does not report variance or sensitivity across runs. Only averaged point estimates appear in all result tables."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B clearly states the repetition methodology: 100 test inputs sampled, 10 program variants sampled, with combinatorial selection (e.g., C(10,4)=210 rounds for k=4)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The number of variants k is varied (2-10) as a parameter study but no search budget for other design decisions is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports results for all k values (2, 4, 6, 8, 10) rather than cherry-picking the best. The paper uses three comparison methods (Average, Best vs. Best, Worst vs. Worst) for comprehensive evaluation."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implemented Differential Prompting Plus (DPP) themselves as a modified version of the original DP method. They do not acknowledge the potential bias of re-implementing baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison of compute cost between methods. TrickCatcher and DPP both generate variants and inputs but may differ in total API calls; this is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether TrickyBugs and EvalPlus adequately capture the general problem of detecting bugs in plausible programs. Both are competition-level programming problems, which may not represent real-world software."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All methods use the same LLM (gpt-3.5-turbo-0125) with different prompting strategies in a non-agentic pipeline."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section notes that 'the TrickyBugs dataset we used was released after gpt-3.5-turbo-0125,' addressing temporal ordering between training data and benchmark."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context. For instance, providing the PUT alongside the specification could leak information about the expected solution structure."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether TrickyBugs tasks or EvalPlus tasks share structural similarities with LLM training data, or whether tasks within the datasets are independent of each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The paper argues against contamination conceptually (temporal ordering, license restrictions, poor baseline performance) but uses no canary strings, membership inference, or n-gram overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "TrickCatcher achieves recall, precision, and F1 scores that are 1.80×, 2.65×, and 1.66× those of the state-of-the-art baseline (DPP).",
       "evidence": "Table 1 shows TrickCatcher at k=10 achieves F1 of 41.31%, 42.35%, and 51.34% on TrickyBugs (C++), TrickyBugs (Python), and EvalPlus, compared to DPP's best F1 of 24.95%, 36.20%, and 35.76% (Section 6.1).",
       "supported": "moderate"
     },
     {
       "claim": "TrickCatcher generates up to 16× fewer false positives for correct programs compared to state-of-the-art methods.",
       "evidence": "Figure 5 (Section 6.2) shows false positive counts on EvalPlus correct programs: TrickCatcher generates 2.57-5.00 FPs vs DPP's 26.33-29.47 FPs.",
       "supported": "moderate"
     },
     {
       "claim": "Each component of TrickCatcher (PUT-guided program generation, generator-based input generation, diversity-driven differential testing) contributes to its final performance.",
       "evidence": "Table 2 (Section 6.3) ablation study on TrickyBugs (C++) shows systematic degradation when components are replaced with basic alternatives (6 patterns compared).",
       "supported": "moderate"
     },
     {
       "claim": "TrickCatcher's performance remains consistently stable and high with different numbers of program variants.",
       "evidence": "Figure 6 (Section 6.4) shows TrickCatcher's precision and F1 remain stable as k changes from 2-10 on TrickyBugs (C++), unlike DPP which fluctuates.",
       "supported": "moderate"
     },
     {
       "claim": "TrickCatcher demonstrates more significant improvement over DPP on harder coding tasks.",
       "evidence": "Figures 7-8 (Section 6.5) show that on TrickyBugs, TrickCatcher performs comparably to DPP on easy tasks but clearly outperforms on difficult ones, with program variants having higher base-test passing rates on hard tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Buggy program variants can also contribute to generating true positive test cases (23.2% in TrickyBugs, 15.0% in EvalPlus of useful variants are buggy).",
       "evidence": "Section 7.1 reports that 23.2% (TrickyBugs) and 15.0% (EvalPlus) of variants that produced correct oracles for TP test cases were actually buggy.",
       "supported": "moderate"
     },
     {
       "claim": "TrickCatcher generalizes across different language models, with stronger models yielding better performance.",
       "evidence": "Table 3 (Section 7.2) shows deepseek-v3 achieves 59.54% F1 on EvalPlus vs gpt-3.5-turbo's 51.34%, both using TrickCatcher.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification despite averaged results",
       "detail": "The paper describes an elaborate combinatorial repetition methodology (Appendix B) but reports only point estimates in all tables. With 210+ rounds per configuration, standard deviations could easily be reported but are absent, making it impossible to assess whether observed differences are meaningful or within noise."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Claims of 'significantly outperforming' baselines are based purely on comparing averaged numbers. No statistical tests are performed despite the repetition methodology producing enough data for them."
     },
     {
       "flag": "Self-implemented modified baseline",
       "detail": "The main baseline DPP is a self-modified version of Differential Prompting ('Differential Prompting Plus'), adapted by the authors for plausible programs. The original DP was not designed for this setting. The authors acknowledge modifying it but do not discuss whether their implementation may underperform the original authors' intent."
     },
     {
       "flag": "Ablation on single dataset only",
       "detail": "The ablation study (Table 2) is reported only on TrickyBugs (C++) 'due to page limit.' Component contributions may differ across datasets and languages, but this is not verified."
     },
     {
       "flag": "No cost reporting despite budget-constrained model choice",
       "detail": "The paper chose gpt-3.5-turbo specifically due to budget constraints but never quantifies the actual cost. The method requires generating 10 program variants + input generators per program, making cost a key practical consideration."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational benchmark (HumanEval/Codex) for LLM code generation evaluation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "EvalPlus benchmark used as one of the two evaluation datasets; addresses rigor in LLM code evaluation."
     },
     {
       "title": "Nuances are the key: Unlocking ChatGPT to find failure-inducing tests with differential prompting",
       "authors": ["Tsz-On Li", "Wenxi Zong", "Yibo Wang"],
       "year": 2023,
       "relevance": "State-of-the-art baseline (Differential Prompting) for LLM-based test case generation for bug detection."
     },
     {
       "title": "TrickyBugs: A dataset of corner-case bugs in plausible programs",
       "authors": ["Kaibo Liu", "Yudong Han", "Yiyang Liu", "Jie M. Zhang"],
       "year": 2024,
       "relevance": "Primary evaluation dataset of human-written plausible programs with tricky bugs from online judge platforms."
     },
     {
       "title": "Who judges the judge: An empirical study on online judge tests",
       "authors": ["Kaibo Liu", "Yudong Han", "Jie M. Zhang", "Zhenpeng Chen"],
       "year": 2023,
       "relevance": "Identifies 3,440 tricky bugs in human-written programs on online judge platforms, motivating the TrickCatcher approach."
     },
     {
       "title": "Large language model-based agents for software engineering: A survey",
       "authors": ["Junwei Liu", "Kaixin Wang", "Yixuan Chen"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Survey of LLM-based agents for software engineering tasks including test generation."
     },
     {
       "title": "Evaluating and improving ChatGPT for unit test generation",
       "authors": ["Zhiqiang Yuan", "Mingwei Liu", "Shiji Ding"],
       "year": 2024,
       "relevance": "Evaluates LLM-based unit test generation (ChatTester), a related approach in the test generation space."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "relevance": "Empirical evaluation of LLMs (TestPilot) for automated unit test generation."
     },
     {
       "title": "Code-aware prompting: A study of coverage-guided test generation in regression setting using LLM",
       "authors": ["Gabriel Ryan", "Siddhartha Jain", "Mingyue Shang"],
       "year": 2024,
       "relevance": "Coverage-guided LLM test generation (SymPrompt), a related approach combining code analysis with prompting."
     },
     {
       "title": "The counterfeit conundrum: Can code language models grasp the nuances of their incorrect generations?",
       "authors": ["Alex Gu", "Wen-Ding Li", "Naman Jain"],
       "year": 2024,
       "relevance": "Studies whether code LLMs can distinguish correct from incorrect generated code, directly related to plausible program detection."
     },
     {
       "title": "B4: Towards optimal assessment of plausible code solutions with plausible tests",
       "authors": ["Mouxiang Chen", "Zhongxin Liu", "He Tao"],
       "year": 2024,
       "relevance": "Addresses assessment of plausible code solutions, closely related to the plausible program testing problem."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners working with LLM-generated code could use differential testing to find bugs, though the approach currently requires clear specifications."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The diversity-driven oracle (trusting minority outputs over majority voting) is mildly counterintuitive but not paradigm-shifting."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; this is a testing/debugging tool."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims about existing methods or institutions."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with data, but not a pip-installable tool or live demo."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Academic authors from Peking University and UCL; Mark Harman is well-known in software testing but not a household name."
     }
   }
 }