ai-research-survey

scan.json (30438B)

---

 {
   "paper": {
     "title": "Large Language Models for Fault Localization: An Empirical Study",
     "authors": [
       "Yingjian Xiao",
       "Weiwei Gong",
       "Rongqun Hu",
       "Hongwei Li",
       "Anquan Jie"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.20521",
     "doi": "10.48550/arXiv.2510.20521"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Gemini-2.5-flash consistently outperforms GPT-4.1 mini, Qwen2.5-coder-32b-instruct, and DeepSeek-V3 on statement-level fault localization across both HumanEval-Java and Defects4J datasets. Providing bug report context yields the largest performance boost (e.g., Gemini Top@5 jumps from 6.08% to 23.67% on Defects4J). Few-shot learning shows diminishing returns beyond 2 examples, and chain-of-thought reasoning is model-dependent—it degrades GPT-4.1 mini on HumanEval-Java but substantially improves DeepSeek-V3 on Defects4J.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No code repository, GitHub link, or archive is provided anywhere in the paper. The experimental scripts, prompt templates, and evaluation pipeline are not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two publicly available datasets: HumanEval-Java (164 bugs, reference [28]) and Defects4J v1.2.0 (395 bugs). Both are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependency lists, or hardware details are provided. The paper does not describe what system the experiments ran on or what software versions were used."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or step-by-step guide is provided. A reader could not reproduce the experiments without substantial guesswork about prompts, parsing, and evaluation pipeline."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main performance tables (Tables 3–8) report only point estimates for Top@k and Pass@k. No confidence intervals or error bars accompany the primary accuracy results. Box plots appear only in the time overhead analysis."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses Scott-Knott ESD tests for time overhead analysis (Section 4.4.1), but the core performance comparison claims in RQ1–RQ3 (e.g., 'Gemini-2.5-flash performs best') are made purely by comparing numbers in tables without any significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 3–8 report absolute percentages for all models and conditions, allowing readers to compute the magnitude of differences (e.g., Gemini Top@5=65.03% vs Qwen=46.63% on HumanEval-Java, and with/without bug report comparisons on Defects4J)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for using 164 bugs (HumanEval-Java) or 395 bugs (Defects4J). No discussion of whether these sizes are sufficient for the statistical claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs 13 rounds per experiment and computes Pass@k, but does not report standard deviation, IQR, or any spread measure for the main accuracy results. Box plots in the time analysis show distributions, but the core performance tables are point estimates only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Zero-shot standard prompting serves as the baseline, with Few-shot and CoT strategies compared against it. Multiple models are compared against each other across all conditions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All four models are recent (2024–2025 releases per Table 2): GPT-4.1 mini (Apr 2025), Qwen2.5-coder-32b-instruct (Nov 2024), Gemini-2.5-flash (Apr 2025), DeepSeek-V3 (Dec 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 tests with/without bug report context, RQ2 systematically varies shot count (1/2/3), and RQ3 adds/removes CoT reasoning. These controlled variations show which prompting components contribute to performance."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five metrics are used: Top@5, Top@10, Pass@1, Pass@5, Pass@10 (Section 3.4)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of model outputs is performed. Evaluation is entirely automated via exact matching of predicted fault locations against ground truth."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Models are evaluated in zero-shot/few-shot mode without fine-tuning. The full datasets (HumanEval-Java, Defects4J) serve as test sets that were not used for model training or selection."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model, dataset, prompting strategy (zero-shot/few-shot/CoT), and shot count (1/2/3). Defects4J results are further split by with/without bug report. Time analysis separates successful vs failed localizations."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where approaches fail: CoT degrades GPT-4.1 mini performance on HumanEval-Java, 3-shot causes regression for some models, and failed localizations take significantly longer (Section 4.4.1). Finding 1 of RQ3 explicitly discusses when CoT does not help."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: CoT hurts GPT-4.1 mini and DeepSeek-V3 on HumanEval-Java (Table 7), 3-shot degrades performance for multiple models, and increasing examples shows diminishing or negative returns (RQ2 findings)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that bug report context significantly improves performance (supported by Table 4), few-shot has diminishing returns (supported by Tables 5–6), and CoT depends on model reasoning ability (supported by Tables 7–8). All are backed by experimental data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'providing bug report context significantly improves performance' are supported by controlled single-variable manipulation (with vs without bug report, same models and dataset). Similarly, few-shot and CoT claims are based on adding/removing specific prompt components while holding other variables constant."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Large Language Models for Fault Localization' is broad, but the study tests only 4 models on 2 Java datasets. While the threats section notes Java-only limitation, the title and findings language ('大语言模型在缺陷定位任务中的性能特征') generalizes beyond the tested scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats section discusses methodological concerns (prompt bias, API variability, strict matching) but does not discuss alternative explanations for specific findings. For example, no discussion of why Gemini outperforms (training data? architecture? inference strategy?) or why CoT helps some models but not others beyond stating it 'depends on reasoning ability.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures statement-level fault localization accuracy (Top@k, Pass@k) and claims to evaluate fault localization ability. The measurements directly correspond to the claims without proxy gaps."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named (GPT-4.1 mini, Qwen2.5-coder-32b-instruct, Gemini-2.5-flash, DeepSeek-V3) with release dates and parameter counts in Table 2, but no API version identifiers or snapshot dates are provided. The schema explicitly states marketing names like 'Gemini-2.5' without a snapshot date do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompting strategies (Standard, Few-shot, CoT) at a conceptual level but never provides the actual prompt text used in experiments. Readers cannot reconstruct the exact prompts sent to the models."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs (n=13) and k values (1, 5, 10) are stated, but model inference hyperparameters (temperature, top-p, max tokens) are not reported anywhere in the paper."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper sends prompts directly to model APIs without any multi-step reasoning framework, tool use, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how buggy code files were formatted into prompts, how bug reports were extracted and formatted for Defects4J, how model outputs were parsed, or how the exact-match evaluation was implemented at the code level."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 ('THREATS TO VALIDITY') provides substantive discussion of internal, construct, and external validity threats across three subsections."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section includes specific concerns: prompt wording bias, strict complete-match criterion potentially overestimating error rates, format-parsing false negatives (missing brackets/semicolons), potential data leakage despite newer datasets, and coverage of only Java and limited defect types."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "External validity section explicitly states results are limited to Java and two specific datasets, that models evolve quickly and may invalidate findings, and that tested defect types do not cover complex industrial defects like concurrency bugs or security vulnerabilities."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental outputs (model responses, parsed results, per-bug success/failure logs) are released. Only aggregated metrics appear in the tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Both datasets are well-characterized: HumanEval-Java (164 manually injected Java bugs with JUnit tests, from reference [28]) and Defects4J v1.2.0 (395 bugs from 6 open-source projects with bug reports and tests). Table 1 summarizes dataset properties."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard public benchmarks (HumanEval-Java, Defects4J) as data sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw datasets to final metrics is not documented. Missing details include: how code was extracted and formatted into prompts, how model responses were parsed, how line-number matching was performed, and how Pass@k was computed from the 13 rounds."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, grants, or acknowledgments section appears in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Nanchang Institute of Technology (Computer Information Engineering School) and Jiangxi Normal University (School of Artificial Intelligence). No authors are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot determine funder independence because no funding is disclosed. The absence of a funding disclosure does not confirm independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 lists knowledge cutoff dates for all four models: GPT-4.1 mini (2024-06), Qwen2.5-coder-32b-instruct (2023-10), Gemini-2.5-flash (2025-01), DeepSeek-V3 (2024-07)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes HumanEval-Java was specifically created to avoid data leakage (Section 3.2) and acknowledges in Section 5 that 'data leakage cannot be completely ruled out' for existing benchmarks whose code may have appeared in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper deliberately selects HumanEval-Java because it was designed to prevent contamination. For Defects4J (2018, predating all model cutoffs), the construct validity threats acknowledge that 'if test data or similar code fragments appeared in model training data, the model may exhibit overly strong generalization.'"
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLMs on code datasets."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates models on public code benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4.2 and Tables 9–10 report per-round API costs for each model on each dataset, including input/output token counts and pricing. Time overhead per inference is also reported in Section 4.4.1."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-call costs are reported but the total computational budget (total API spend across all 13 rounds × all conditions × all models) is not stated. No hardware specifications are provided."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs 13 rounds per experiment to compute Pass@k, but does not report result variation across runs or discuss seed sensitivity. No standard deviation or spread measure accompanies the main accuracy metrics."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.4 explicitly states n=13 total samples per experiment and k=1,5,10 for the Pass@k computation."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper does not discuss how prompt templates were designed or whether alternative templates were tried. Temperature and other inference parameters are not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explain how the specific prompt templates, few-shot examples, or CoT instructions were selected. The threats section acknowledges that 'the choice of examples, subtle wording differences, and chain-of-thought guidance' may affect results, but no selection process is documented."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous pairwise comparisons across 4 models × multiple settings but applies no correction for multiple comparisons. Scott-Knott ESD is used only for time analysis grouping, not for the main accuracy claims."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The authors evaluate third-party models (GPT, Qwen, Gemini, DeepSeek) they did not create. No self-comparison bias exists since no novel system is proposed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 explicitly analyzes time overhead and API cost alongside accuracy results. Tables 9–10 report per-model costs, and the time analysis (Figures 1–2) shows inference time distributions, enabling cost-performance comparisons."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether HumanEval-Java (synthetic bugs) or Defects4J (historical bugs from 6 projects) adequately measure real-world fault localization ability. The difference in results between the two benchmarks (65% vs 24% Top@5 for Gemini) suggests a validity gap, but this is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is used. Models are prompted directly via APIs with no multi-step framework or tool integration."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper lists model training cutoffs (Table 2) and explicitly selects HumanEval-Java because it was created in September 2023 to avoid data leakage. The construct validity section acknowledges that Defects4J (2018) predates all model cutoffs, creating leakage risk."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the prompt format (providing full code, error messages, or bug reports) leaks information beyond what would be available in realistic debugging scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval-Java examples share structural patterns or whether Defects4J bugs from the same project are independent. Non-independence between same-project bugs could inflate reported metrics."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap) is applied. The paper relies only on the temporal argument that HumanEval-Java was designed to avoid contamination."
       }
     }
   },
   "claims": [
     {
       "claim": "Gemini-2.5-flash outperforms all other models on both datasets, demonstrating strong adaptability and generalization for fault localization.",
       "evidence": "Tables 3–4: Gemini achieves Top@5=65.03% on HumanEval-Java and Top@5=23.67% (with bug report) on Defects4J, highest among all models in all metrics.",
       "supported": "moderate"
     },
     {
       "claim": "Bug report context significantly improves fault localization performance across all models.",
       "evidence": "Table 4: All models show large improvements with bug report context on Defects4J (e.g., Gemini Top@5 from 6.08% to 23.67%, GPT-4.1 mini from 3.90% to 15.15%).",
       "supported": "strong"
     },
     {
       "claim": "Few-shot learning shows diminishing returns, with 2-shot typically being optimal and 3-shot sometimes degrading performance.",
       "evidence": "Tables 5–6: GPT-4.1 mini peaks at 2-shot (56.44% Top@10) then drops at 3-shot (51.81%); Qwen degrades steadily from 1-shot onward. DeepSeek-V3 is an exception, peaking at 3-shot.",
       "supported": "moderate"
     },
     {
       "claim": "Chain-of-thought reasoning effectiveness is model-dependent: it helps some models on complex defects but degrades others.",
       "evidence": "Tables 7–8: CoT degrades GPT-4.1 mini on HumanEval-Java (Top@5 drops from 50% to 34.36%) but improves DeepSeek-V3 on Defects4J (Top@5 rises from 11.56% to 19.11%). Gemini is stable.",
       "supported": "moderate"
     },
     {
       "claim": "Failed localization tasks take significantly longer, especially on the Defects4J dataset.",
       "evidence": "Section 4.4.1 with box plots: Failed tasks show higher median inference times across all models. DeepSeek-V3 averages 22.96s on failed Defects4J tasks vs 3–9s on successes.",
       "supported": "moderate"
     },
     {
       "claim": "Closed-source models generally have higher API costs than open-source models.",
       "evidence": "Tables 9–10 report per-round costs. On Defects4J, GPT-4.1 mini costs $1.917 and Gemini $1.344, while Qwen costs ¥8.642 (~$1.19) and DeepSeek ¥10.541 (~$1.45). The cost difference is modest and currency-dependent.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Prompts not provided",
       "detail": "The actual prompt text for Standard, Few-shot, and CoT strategies is never shown. Readers cannot verify what the models were actually asked, making reproduction impossible. Prompt design is acknowledged as a threat to validity."
     },
     {
       "flag": "No code or artifacts released",
       "detail": "No experimental code, evaluation scripts, or raw results are released. Combined with missing prompts, this makes the study completely non-reproducible."
     },
     {
       "flag": "No significance tests on main results",
       "detail": "The core performance comparisons (RQ1–RQ3) rely entirely on comparing numbers in tables. Claims like 'Gemini outperforms all models' have no statistical backing. Scott-Knott ESD is used only for time overhead analysis."
     },
     {
       "flag": "Missing inference hyperparameters",
       "detail": "Temperature, top-p, and max tokens are not reported for any model. These settings significantly affect LLM output and could explain performance differences."
     },
     {
       "flag": "Duplicate references",
       "detail": "References [6] and [18] are the same paper (Wu et al. 2023), and references [8]/[9] and [19]/[24] overlap. The paper itself notes '[6] 和18 相同', suggesting incomplete editing."
     },
     {
       "flag": "Few-shot example selection not described",
       "detail": "The paper states few-shot examples were 'carefully selected' but does not describe the selection criteria, source, or whether they come from the same dataset being tested. This could introduce data leakage or selection bias."
     }
   ],
   "cited_papers": [
     {
       "title": "Software testing with large language models: Survey, landscape, and vision",
       "authors": ["Junjie Wang", "Yuchao Huang", "Chunyang Chen"],
       "year": 2024,
       "relevance": "Comprehensive survey of LLMs for software testing tasks, directly relevant to understanding LLM capabilities in code-related tasks."
     },
     {
       "title": "Large language models in fault localisation",
       "authors": ["Yonghao Wu", "Zheng Li", "Jie M. Zhang"],
       "year": 2023,
       "arxiv_id": "2308.15276",
       "relevance": "Directly evaluates LLMs (ChatGPT) for fault localization on Defects4J, the most closely related prior work."
     },
     {
       "title": "Large language models for test-free fault localization",
       "authors": ["Aidan Z. H. Yang", "Claire Le Goues", "Ruben Martins"],
       "year": 2024,
       "relevance": "Proposes LLMAO framework applying LLMs to fault localization without test cases, demonstrating LLM potential in this domain."
     },
     {
       "title": "Evaluating fault localization and program repair capabilities of existing closed-source general-purpose LLMs",
       "authors": ["Shuai Jiang", "Jie Zhang", "Wei Chen"],
       "year": 2024,
       "relevance": "Systematic comparison of closed-source LLMs (GPT-4, Claude) for fault localization and repair, closely related evaluation methodology."
     },
     {
       "title": "A quantitative and qualitative evaluation of LLM-based explainable fault localization",
       "authors": ["Sungmin Kang", "Gabin An", "Shin Yoo"],
       "year": 2024,
       "relevance": "Evaluates LLM-based fault localization with chain-of-thought reasoning, directly relevant to this paper's RQ3."
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought prompting, the core technique studied in this paper's RQ3."
     },
     {
       "title": "Language models are Few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational work on few-shot learning with LLMs, the core technique studied in this paper's RQ2."
     },
     {
       "title": "Hybrid automated program repair by combining large language models and program analysis",
       "authors": ["Fengjie Li", "Jiajun Jiang", "Jia Sun"],
       "year": 2025,
       "relevance": "GiantRepair method combining LLMs with program analysis for automated program repair, demonstrating LLM-augmented debugging."
     },
     {
       "title": "Context-aware prompting for LLM-based program repair",
       "authors": ["Yuxiang Li", "Muchen Cai", "Jianan Chen"],
       "year": 2025,
       "relevance": "CodeCorrector uses test failure information to construct targeted prompts for LLM-based program repair, relevant to prompting strategy design."
     },
     {
       "title": "ThinkRepair: Self-directed automated program repair",
       "authors": ["Xin Yin", "Chao Ni", "Shaohua Wang"],
       "year": 2024,
       "relevance": "Two-phase automated repair using few-shot learning and iterative testing with LLMs, demonstrating reasoning-enhanced repair workflows."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier"],
       "year": 2023,
       "relevance": "Introduces HumanEval-Java dataset used in this study, designed specifically to avoid data leakage in LLM evaluation."
     },
     {
       "title": "Gitbug-java: A reproducible benchmark of recent java bugs",
       "authors": ["André Silva", "Nuno Saavedra", "Martin Monperrus"],
       "year": 2024,
       "relevance": "New Java fault localization benchmark with better reproducibility and temporal properties, relevant to evaluation methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners choosing LLMs for automated debugging could use the model comparison and prompting strategy findings to guide selection."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that chain-of-thought reasoning can degrade performance is mildly surprising but not strongly contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk implications in this study of debugging tool accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward empirical comparison."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, tool, or demo is released. Results cannot be tried or replicated."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Evaluates well-known models (GPT, Gemini, DeepSeek) but authors are from lesser-known Chinese universities."
     }
   }
 }