ai-research-survey

scan.json (31124B)

---

 {
   "paper": {
     "title": "LLM-Powered Test Case Generation for Detecting Bugs in Plausible Programs",
     "authors": [
       "Kaibo Liu",
       "Zhenpeng Chen",
       "Yiyang Liu",
       "Jie M. Zhang",
       "Mark Harman",
       "Yudong Han",
       "Yun Ma",
       "Yihong Dong",
       "Ge Li",
       "Gang Huang"
     ],
     "year": 2024,
     "venue": "Annual Meeting of the Association for Computational Linguistics",
     "arxiv_id": "2404.10304",
     "doi": "10.18653/v1/2025.acl-long.20"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TrickCatcher, an LLM-powered test case generation approach combining PUT-guided program variant generation, generator-based input generation, and diversity-driven differential testing, achieves F1 scores of 41.31%, 42.35%, and 51.34% on TrickyBugs (C++), TrickyBugs (Python), and EvalPlus respectively, significantly outperforming the best baseline (Differential Prompting Plus) with up to 1.66× F1 improvement. The approach generates up to 16× fewer false positives than baselines on correct programs. An ablation study confirms each of the three components contributes meaningfully, and the counterintuitive diversity-driven oracle selection (trusting outputs that differ from the PUT) outperforms traditional majority voting.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code and data used are available at https://github.com/RinCloud/TrickCatcher' and provides a working GitHub URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets used are publicly available: TrickyBugs (MIT license) and EvalPlus (Apache 2.0 license). The paper also states code and data are available at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications (requirements.txt, Dockerfile, library versions) are mentioned in the paper. Only the LLM model name and the CYaRon Python library are mentioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The methodology is described algorithmically but there is no 'Reproducing Results' section or equivalent."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and all results report only point estimates (e.g., '41.31%' F1) with no confidence intervals or error bars, despite the authors performing multiple repetitions."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TrickCatcher 'significantly outperforming' baselines (Section 6.1) but provides no statistical significance tests (no p-values, t-tests, or similar). Claims of superiority are based solely on comparing point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative improvements with context: '1.80×, 2.65×, and 1.66× those of the state-of-the-art baseline' and shows absolute baseline values alongside TrickCatcher values in Table 1, allowing readers to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample sizes (366 human-written and 151 AI-generated programs). The datasets are used as-is without discussion of whether these sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite performing combinatorial repetitions (Appendix B describes C(k,10) rounds and averaging over 100 inputs), no standard deviation, IQR, or other spread measure is reported in any table or figure. Only averaged values are shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: DirectChat (CHAT), Differential Prompting Plus (DPP), and Automated Program Repair (APR), described in Section 5.4."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Differential Prompting (Li et al., 2023) is described as 'the state-of-the-art in LLM-based test case generation.' The baselines are from 2023, which is recent relative to the 2024 submission."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 6.3, Table 2) presents a thorough ablation study with 6 patterns combining different variants of program generation, input generation, and differential testing components."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: recall, precision, and F1 score, all reported in Table 1 and throughout the evaluation."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated using canonical programs and checkers. While input validity for TrickyBugs was 'manually verified' (Section 5.3), there is no human evaluation of the system's test case outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "TrickyBugs and EvalPlus are established benchmarks with canonical solutions and additional test cases. The datasets provide ground truth for evaluation that is separate from any development data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (TrickyBugs C++, TrickyBugs Python, EvalPlus) in Table 1, by task difficulty (RQ5, Figures 7-8), and across different numbers of program variants k."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 (Section 6.2, Figure 5) analyzes false positives and their categories (incorrect oracles vs. invalid inputs). Section 7.1 discusses buggy program variants. RQ5 discusses where TrickCatcher has less advantage."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that 40.10% of directly generated test inputs are invalid (Section 1), that recall can decrease with more variants for DPP (Table 1), and that TrickCatcher's recall on TrickyBugs Python is actually slightly lower than DPP in the best-vs-best comparison (-10.61%)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract's claims of 1.80×, 2.65×, and 1.66× recall/precision/F1 improvements are supported by Table 1's best-vs-best comparisons. F1 scores of 41.31%, 42.35%, and 51.34% are verified in the same table."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2, RQ3) provides controlled single-variable manipulation across 6 patterns, systematically varying one component at a time to justify claims like 'each component of TrickCatcher contributes meaningfully to its overall performance.'"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are bounded to the tested datasets and languages. The title 'Detecting Bugs in Plausible Programs' accurately describes the scope. Section 7.2 tests generalization to deepseek-v3 but frames it appropriately as verifying 'generalization capability' rather than claiming universal applicability."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why TrickCatcher outperforms baselines. For example, it does not consider whether the improvement is primarily due to having access to the PUT (which DPP does not) vs. the other design choices, nor whether the results might be explained by the specific characteristics of the evaluation datasets."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures TP, FP, precision, recall, and F1 for bug detection, which directly maps to the claimed capability. No proxy gap exists — 'detecting bugs in plausible programs' is exactly what is measured."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 specifies 'gpt-3.5-turbo-0125' as the exact model version, including the snapshot date suffix. deepseek-v3 is also named in Section 7.2."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figures 3 and 4 provide the actual prompt templates used for program variant generation and test input generator generation, including the instruction text and placeholders clearly labeled."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM API hyperparameters (temperature, top-p, max tokens, sampling settings) are reported anywhere in the paper despite multiple LLM calls being central to the approach."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "TrickCatcher is a sequential pipeline (generate variants → generate inputs → differential testing), not an agentic scaffolding system. There is no retry logic, feedback loops, memory, or tool use."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes how programs were selected: TrickyBugs provides 251 C++ and 115 Python plausible programs; EvalPlus programs were filtered from pre-generated LLM code samples to obtain those passing base tests but failing extra tests, yielding 151 tasks."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after the Conclusion, discussing three specific limitations: model budget constraints, LLM uncertainty, and data leakage risk."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section identifies specific threats: (1) only two models used due to budget, suggesting more advanced LLMs could improve results; (2) LLM behavior uncertainty mitigated by multiple repetitions and averaging; (3) data leakage addressed by noting TrickyBugs was released after the model and EvalPlus prohibits training use."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. The Limitations mention budget constraints and suggest future work but do not clearly bound what claims should not be extrapolated from these results."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Code and data are available at https://github.com/RinCloud/TrickCatcher. Both source datasets (TrickyBugs, EvalPlus) are publicly available under MIT and Apache 2.0 licenses."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.2 describes both datasets: TrickyBugs contains programs from an online judge platform, and EvalPlus programs were filtered from pre-generated LLM code samples with specific inclusion criteria (pass base tests, fail extra tests)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data comes from standard benchmarks (TrickyBugs and EvalPlus)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: for EvalPlus, programs are filtered from pre-generated samples to find those passing base but failing extra tests (resulting in 151 tasks). For TrickyBugs, 251 C++ and 115 Python plausible programs are selected. Appendix B documents the repetition and averaging pipeline."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgements section lists specific grants: National Key R&D Program (2023YFB4503801), NSFC grants (62192733, 62192730), Hubei Province program (2023BAA024), and ITEA projects for Jie M. Zhang."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: Peking University, Nanyang Technological University, King's College London, University College London, and National Key Laboratory of Data Space Technology and System."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from government research programs (NSFC, National Key R&D, Hubei Province, InnovateUK) with no commercial interest in the evaluation outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff date for gpt-3.5-turbo-0125 or deepseek-v3. The Limitations section mentions TrickyBugs was released after the model but does not state the actual cutoff."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section discusses this: 'the TrickyBugs dataset we used was released after gpt-3.5-turbo-0125, and EvalPlus explicitly prohibits its use for training LLMs. Moreover, the poor performance of the three LLM-based baselines further suggests that data leakage is not a main concern.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section addresses contamination: TrickyBugs released after the model, EvalPlus prohibits training use, and poor baseline performance is cited as evidence against leakage being a factor."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on benchmark datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates automated test case generation on code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference costs, API costs, or latency figures are reported despite the approach requiring hundreds of LLM calls per PUT (generating 10 variants + input generators + 100 test inputs)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper mentions 'budget constraints' as motivation for using gpt-3.5-turbo but does not quantify the actual spend or compute used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "While Appendix B describes a combinatorial repetition scheme, no variance or sensitivity across runs is reported. All tables show only averaged point estimates."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B explicitly states: 100 test inputs sampled, 10 program variants sampled, with C(k,10) combinatorial rounds. For CHAT: 100 test cases sampled. For APR: 10 patches sampled."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No mention of any hyperparameter search. The number of program variants k is varied as a parameter study, but no search budget for other hyperparameters (e.g., prompt design, number of inputs) is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported for all k values (2, 4, 6, 8, 10) in Table 1, with multiple comparison methods (Average, Best vs. Best, Worst vs. Worst). No cherry-picking of a single best configuration."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own version of the DPP baseline (modified to DPP+ for fair comparison) and do not acknowledge the bias of evaluating their own system against their own re-implementation of the baseline."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "TrickCatcher involves additional compute over DPP (PUT-guided generation, input generator creation + execution) but no comparison of compute budgets between methods is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether TrickyBugs and EvalPlus actually represent real-world bug detection scenarios, or whether plausible programs from competitive programming are representative of bugs encountered in production software."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "All methods use the same underlying LLM (gpt-3.5-turbo-0125) with different pipelines. The pipeline IS the thing being tested, so no scaffold confound exists."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section notes that 'the TrickyBugs dataset we used was released after gpt-3.5-turbo-0125,' directly addressing temporal ordering between model training and benchmark availability."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For instance, providing the PUT alongside the specification to the LLM could be considered a form of information leakage compared to real-world scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether programs in TrickyBugs or EvalPlus share structural similarities, come from overlapping sources, or whether the competitive programming tasks are independent of each other."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. The contamination discussion is entirely conceptual (dataset release dates, license terms, baseline performance argument)."
       }
     }
   },
   "claims": [
     {
       "claim": "TrickCatcher achieves recall, precision, and F1 scores that are 1.80×, 2.65×, and 1.66× those of the state-of-the-art baseline (DPP).",
       "evidence": "Table 1 shows best-vs-best comparisons across TrickyBugs (C++, Python) and EvalPlus datasets. F1 scores: 41.31%, 42.35%, 51.34% vs DPP's 24.95%, 36.20%, 35.76%.",
       "supported": "strong"
     },
     {
       "claim": "TrickCatcher generates up to 16× fewer false positives for correct programs compared to baselines.",
       "evidence": "Figure 5 (RQ2) shows FP counts on EvalPlus canonical programs: TrickCatcher (k=2) generates 2.57 FPs vs DPP (k=10) at 26.33+1.67 FPs. The 16× figure appears to compare specific configurations.",
       "supported": "strong"
     },
     {
       "claim": "Each component of TrickCatcher (PUT-guided generation, generator-based input generation, diversity-driven differential testing) contributes to performance.",
       "evidence": "Table 2 ablation study (RQ3) with 6 patterns on TrickyBugs (C++) shows systematic improvement when adding each component, comparing patterns 2 vs 3 (differential testing), 4 vs 6 (input generation), and 3/5 vs 6 (program generation).",
       "supported": "strong"
     },
     {
       "claim": "TrickCatcher's performance remains consistently stable with different numbers of program variants.",
       "evidence": "Figure 6 (RQ4) shows TrickCatcher's precision and F1 remain stable across k=2 to k=10, while DPP fluctuates significantly.",
       "supported": "moderate"
     },
     {
       "claim": "TrickCatcher demonstrates more significant improvement over DPP on more difficult coding tasks.",
       "evidence": "Figures 7 and 8 (RQ5) show difficulty distribution comparisons and program variant passing rates grouped by difficulty. TrickCatcher's advantage is more pronounced on high-difficulty tasks.",
       "supported": "moderate"
     },
     {
       "claim": "Generator-based input generation produces valid inputs, eliminating false positives due to invalid inputs.",
       "evidence": "Figure 5 shows TrickCatcher produces zero FPs from invalid inputs across all k values, while 40.10% of directly generated inputs are reported as invalid in Section 1.",
       "supported": "strong"
     },
     {
       "claim": "TrickCatcher generalizes to different language models (deepseek-v3).",
       "evidence": "Table 3 shows deepseek-v3 achieves 59.54% F1 (k=5) on EvalPlus, outperforming gpt-3.5-turbo's 51.34% (k=10), but this is tested on only one dataset with one additional model.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "The paper repeatedly claims TrickCatcher 'significantly outperforms' baselines but provides no statistical significance tests. All comparisons are based on point estimates without p-values, confidence intervals, or any formal test of whether observed differences could be due to chance."
     },
     {
       "flag": "No variance or uncertainty reported despite multiple runs",
       "detail": "Appendix B describes an elaborate combinatorial repetition scheme (C(k,10) rounds × 100 inputs), but no standard deviation, interquartile range, or any spread measure is reported. Readers cannot assess the stability of the results."
     },
     {
       "flag": "Modified baseline may disadvantage comparison",
       "detail": "The primary baseline DPP is a modified version of Differential Prompting (DPP+), adapted by the authors to handle plausible programs. The original method was not designed for this setting, and the modification may not represent the strongest possible adaptation."
     },
     {
       "flag": "No cost analysis despite heavy LLM usage",
       "detail": "The approach calls the LLM to generate 10 variants, create input generators, and produce 100 test inputs per program. With 517 programs across two datasets, this represents thousands of API calls, yet no cost or latency analysis is provided. The paper mentions 'budget constraints' as motivation for gpt-3.5-turbo but never quantifies costs."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "No LLM API hyperparameters (temperature, top-p, max tokens) are reported despite being critical to LLM output quality and reproducibility."
     }
   ],
   "cited_papers": [
     {
       "title": "Nuances are the key: Unlocking ChatGPT to find failure-inducing tests with differential prompting",
       "authors": ["Tsz-On Li", "Wenxi Zong", "Yibo Wang", "Haoye Tian", "Ying Wang", "Shing-Chi Cheung", "Jeff Kramer"],
       "year": 2023,
       "relevance": "State-of-the-art LLM-based test case generation baseline using differential testing, directly compared with TrickCatcher."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus benchmark used for evaluation, relevant to AI code generation quality assessment."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on evaluating LLMs for code generation (Codex/HumanEval), foundational to the benchmark evaluation approach."
     },
     {
       "title": "TrickyBugs: A dataset of corner-case bugs in plausible programs",
       "authors": ["Kaibo Liu", "Yudong Han", "Yiyang Liu", "Jie M. Zhang", "Zhenpeng Chen", "Federica Sarro", "Gang Huang", "Yun Ma"],
       "year": 2024,
       "relevance": "Primary evaluation dataset containing human-written plausible programs with tricky bugs from online judge platforms."
     },
     {
       "title": "Large language model-based agents for software engineering: A survey",
       "authors": ["Junwei Liu", "Kaixin Wang", "Yixuan Chen", "Xin Peng", "Zhenpeng Chen", "Lingming Zhang", "Yiling Lou"],
       "year": 2024,
       "arxiv_id": "2409.02977",
       "relevance": "Comprehensive survey of LLM-based agents in software engineering, providing context for the broader field."
     },
     {
       "title": "Evaluating and improving ChatGPT for unit test generation",
       "authors": ["Zhiqiang Yuan", "Mingwei Liu", "Shiji Ding", "Kaixin Wang", "Yixuan Chen", "Xin Peng", "Yiling Lou"],
       "year": 2024,
       "relevance": "LLM-based test generation approach (ChatTester) compared as related work on using LLMs for software testing."
     },
     {
       "title": "An empirical evaluation of using large language models for automated unit test generation",
       "authors": ["Max Schäfer", "Sarah Nadi", "Aryaz Eghbali", "Frank Tip"],
       "year": 2024,
       "relevance": "Empirical evaluation of LLM-based test generation (TestPilot), directly relevant to the landscape of AI-powered testing."
     },
     {
       "title": "Code-aware prompting: A study of coverage-guided test generation in regression setting using LLM",
       "authors": ["Gabriel Ryan", "Siddhartha Jain", "Mingyue Shang", "Shiqi Wang", "Xiaofei Ma", "Murali Krishna Ramanathan", "Baishakhi Ray"],
       "year": 2024,
       "relevance": "Coverage-guided LLM test generation approach (SymPrompt), related work on using LLMs for test case generation."
     },
     {
       "title": "The counterfeit conundrum: Can code language models grasp the nuances of their incorrect generations?",
       "authors": ["Alex Gu", "Wen-Ding Li", "Naman Jain", "Theo Olausson", "Celine Lee", "Koushik Sen", "Armando Solar-Lezama"],
       "year": 2024,
       "relevance": "Studies whether LLMs can detect bugs in their own generated code, directly relevant to detecting bugs in plausible programs."
     },
     {
       "title": "B4: Towards optimal assessment of plausible code solutions with plausible tests",
       "authors": ["Mouxiang Chen", "Zhongxin Liu", "He Tao", "Yusu Hong", "David Lo", "Xin Xia", "Jianling Sun"],
       "year": 2024,
       "relevance": "Addresses assessment of plausible code solutions using plausible tests, closely related to bug detection in plausible programs."
     },
     {
       "title": "Who judges the judge: An empirical study on online judge tests",
       "authors": ["Kaibo Liu", "Yudong Han", "Jie M. Zhang", "Zhenpeng Chen", "Federica Sarro", "Mark Harman", "Gang Huang", "Yun Ma"],
       "year": 2023,
       "relevance": "Identified 3,440 tricky bugs in human-written programs on online judge platforms, motivating the TrickCatcher approach."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners working on test generation or CI/CD pipelines could use TrickCatcher to detect bugs in plausible programs, and code is publicly available."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The diversity-driven oracle selection (trusting minority outputs over majority voting) is counterintuitive but not a major paradigm challenge."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the work is about improving software testing."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict in the findings."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is available on GitHub at https://github.com/RinCloud/TrickCatcher, though it requires API keys and benchmark data setup."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Mark Harman (UCL) is well-known in software engineering research, and Peking University has strong brand recognition in CS, but not mainstream fame."
     }
   }
 }