ai-research-survey

scan.json (29569B)

---

 {
   "paper": {
     "title": "A Deep Dive into Large Language Models for Automated Bug Localization and Repair",
     "authors": [
       "Soneya Binta Hossain",
       "Nan Jiang",
       "Qiang Zhou",
       "Xiaopeng Li",
       "Wen-Hao Chiang",
       "Yingjun Lyu",
       "Hoan Nguyen",
       "Omer Tripp"
     ],
     "year": 2024,
     "venue": "Proc. ACM Softw. Eng.",
     "arxiv_id": "2404.11595",
     "doi": "10.1145/3660773"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Toggle, a token-granulated bug localization and repair framework, achieves new state-of-the-art on the CodeXGLUE code refinement benchmark (25.07% exact match on Tufano Small) and outperforms existing methods on Defects4J at Top-10/30/50/100 using only a 110M-parameter model. Prompt design has a dramatic effect on bug fixing accuracy—from 16% to 57% for the same model—demonstrating that token-level localization injects strong inductive bias. An adjustment module that bridges tokenizer discrepancies between localization and fixing models consistently improves accuracy across all tested configurations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. Toggle's implementation is not released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: CodeXGLUE (Tufano Small/Medium), CodeReviewer, Defects4J, and the GitHub dataset. All are referenced with citations and are accessible."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions GPU instances and Hugging Face models but does not specify library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The paper describes the methodology in detail but does not include scripts, commands, or a README for reproducing experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1–8 are reported as single point estimates (e.g., '25.07%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'PolyCoder-2.7B reaching the highest accuracy of 25.07%' outperforming the baseline of 23.86% are made by comparing raw numbers with no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with baseline context in all tables (e.g., Table 1 shows baseline accuracies alongside Toggle results, Table 4 shows improvement from Prompt 1 at 16.07% to Prompt 4 at 56.98%), allowing the reader to assess the magnitude of improvements."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for sample sizes. The number of patches per bug (210 for Defects4J) is noted as 'relatively small' compared to prior work but not formally justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 mentions experiments were 'repeated each experiment several times to confirm consistency' but no standard deviations, variance, or spread measures appear in any results table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: NSEdit, CoText for CodeXGLUE; CodeT5, NSEdit for CodeReviewer; CURE, RewardRepair, Recoder, KNOD, Tare, AlphaRepair, TENURE for Defects4J (Tables 1, 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include methods from 2021–2023 (KNOD, Tare, AlphaRepair, TENURE from ICSE 2023), which are contemporary for a 2024 publication."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (four prompts compared), RQ4 (with/without contextual information), and RQ5 (with/without adjustment module) constitute comprehensive ablation studies isolating each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Exact match accuracy is used for CodeXGLUE/CodeReviewer, test-case validation for Defects4J, and Top-K metrics (Top-10, 30, 50, 100) for patch ranking."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "For Defects4J, Section 3.2.1 states: 'we manually executed all corrective patches to ensure they indeed pass all tests and effectively fix the bugs,' constituting manual verification of system outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.1 states datasets are split 80/10/10 for train/validation/test. Defects4J (RQ2) serves as a completely unseen test set not used in fine-tuning."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Tufano Small, Medium, CodeReviewer ±comments), by model backbone (6 LLMs), and by prompt type (4 prompts) across Tables 1, 4, 6, 7, 8."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 shows a concrete failure case where correct localization leads to incorrect fix, and Figure 2 illustrates tokenizer discrepancy failures. Section 3.5.2 discusses why the bug fixing model prematurely closes methods."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "CodeGPT underperforms other models due to Java-only pretraining (Section 3.1.3). Prompt 4 underperforms Prompt 3 on Tufano datasets despite outperforming with ground-truth locations (Table 8 vs Table 4). Location prediction accuracy drops significantly for dual-token prediction (Table 7)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SOTA on CodeXGLUE (supported by Table 1, PolyCoder-2.7B at 25.07% vs NSEdit 23.86%) and better/comparable performance on Defects4J (supported by Table 3, Top-10 through Top-100 rankings)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about prompt design improving accuracy are supported by controlled experiments (RQ3) where only the prompt changes while model, data, and ground-truth locations remain fixed. The adjustment module ablation (RQ5) is similarly controlled."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Large Language Models' broadly but experiments use only 6 models from 110M to 2.7B parameters—far smaller than current frontier models. Datasets are predominantly Java. These scope limitations are not acknowledged in the title or abstract framing."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 (Threats to Validity) offers generic disclaimers: 'there's a possibility that our results may not generalize across other datasets' and 'it is conceivable that [scripts] might contain bugs.' No substantive alternative explanations for the observed results are discussed (e.g., whether improvements are due to reduced sequence length rather than inductive bias)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures exact match accuracy and test-case pass rates, and frames results in terms of 'bug fixing accuracy'—which directly matches the measurements. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with parameter counts are given: CodeGPT-110M (Java checkpoint), CodeParrot-110M (multi-language), CodeGen-350M/2B, PolyCoder-400M/2.7B, CodeT5-large (347M). These are identifiable on Hugging Face."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 provides the complete structure of all four prompt formats with concrete examples showing how buggy code, shared prefix, shared suffix, and separators are arranged. The prompts are deterministically constructed from code, and the construction rules are fully specified."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No training hyperparameters are reported—no learning rate, batch size, number of epochs, optimizer, temperature for generation, or beam size. Only dataset splits (80/10/10) and number of generated patches (70/210) are mentioned."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "Toggle is a pipeline of fine-tuned models (localization → adjustment → fixing), not an agentic scaffold. No tool use, retry logic, or feedback mechanisms are involved."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes dataset characteristics and selection criteria. For Defects4J, 240 single-hunk bugs are selected. The GitHub dataset excludes Defects4J patches via AST comparison. Sections 2.3.1–2.3.3 detail how input prompts are constructed from buggy/fixed code pairs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Threats to Validity' provides a dedicated subsection discussing generalizability, implementation correctness, and accuracy metrics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 states generic threats: 'there's a possibility that our results may not generalize across other datasets' and 'it is conceivable that [tools] might contain bugs.' These are boilerplate disclaimers rather than threats specific to this study's methodology."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not acknowledge that findings may be specific to models in the 110M–2.7B range, to Java code, or to single-hunk bugs."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the benchmark datasets are public, the paper does not release model outputs, generated patches, predicted locations, or fine-tuned model weights. Independent verification of reported numbers is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes each dataset in detail: Tufano Small (58,350 samples) and Medium (65,465) from GitHub commits, CodeReviewer (183,881 multilingual samples), Defects4J (835 bugs from 17 projects, 240 single-hunk used), and the GitHub dataset (1,083,185 commits)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from bug localization to adjustment to bug fixing is described in detail (Sections 2.3.1–2.3.3). Figure 3 provides a visual overview. The adjustment model training pipeline is described step-by-step in Section 2.3.3 with concrete shift ranges (-3 to +3)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned despite 5 of 8 authors being affiliated with Amazon Web Services, suggesting corporate funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Soneya Binta Hossain (University of Virginia), Nan Jiang (Purdue), and Qiang Zhou, Xiaopeng Li, Wen-Hao Chiang, Yingjun Lyu, Hoan Nguyen, Omer Tripp (all Amazon Web Services)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed. With 5/8 authors from AWS, the work is implicitly corporate-funded. While the paper does not evaluate AWS products specifically, the lack of funding disclosure prevents assessment of independence."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The pre-training data cutoff dates for CodeGPT, CodeParrot, CodeGen, PolyCoder, and CodeT5 are not stated. The paper does not mention when these models' training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states the GitHub dataset excluded 'patches associated with the Defects4J project or those resembling any in Defects4J v1.2 or v2.0 were meticulously excluded, based on an AST comparative analysis.' Standard splits are used for other datasets."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While Defects4J overlap with training data is addressed, no discussion of whether CodeXGLUE benchmark examples could appear in the pre-training data of the base models (CodeGPT, CodeParrot, etc.) is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The number of generated patches (210 per bug) is noted as 'relatively small' but no wall-clock time or compute cost is given."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No GPU hours, training time, or total computational budget is stated despite fine-tuning 6 LLMs across 4 datasets with 4 prompt styles (96 experiments) plus adjustment model training."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-run numbers despite Section 5 claiming experiments were 'repeated several times.'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 5 vaguely states 'repeated each experiment several times to confirm consistency' but the exact number of runs is not stated and no variance is shown in results tables."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Training hyperparameters are not even listed, let alone the search process for selecting them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are systematically reported: 6 models × 4 prompts in Table 4, all models on all datasets in Tables 1 and 8. CodeParrot-110M was selected for Defects4J based on 'superior performance over the similarly sized CodeGPT-110M and comparable performance to larger models' (Section 3.2.1)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "No statistical tests are performed at all, let alone corrections for multiple comparisons, despite making dozens of pairwise comparisons across 6 models, 4 prompts, and 4 datasets."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement some baselines (NSEdit on CodeReviewer, Section 3.1.2) and compare their own system against these re-implementations without acknowledging or addressing author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models range from 110M to 2.7B parameters (25× difference) but no performance-per-compute analysis is provided. Finding 1 notes larger models perform better but does not account for the additional compute required."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses exact match as the primary metric, noting only that it is 'commonly used' and 'a more fairer measure of accuracy than either BLEU or CodeBLEU scores' (Section 3.1.1). No deeper discussion of whether exact match captures real-world bug fixing utility."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is involved. Toggle is a deterministic pipeline of fine-tuned models, not a scaffold-based system."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the base models' pre-training data temporally overlaps with the benchmark datasets. CodeXGLUE and Defects4J predate the models' training, creating potential temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information not available in real usage scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 states the GitHub dataset excluded patches 'associated with the Defects4J project or those resembling any in Defects4J v1.2 or v2.0' using AST comparative analysis, directly addressing train-test independence for the Defects4J evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "An AST comparative analysis was used to identify and exclude Defects4J-related patches from the GitHub training dataset (Section 2.2), constituting a concrete decontamination method."
       }
     }
   },
   "claims": [
     {
       "claim": "Toggle achieves new state-of-the-art on CodeXGLUE code refinement benchmark, with PolyCoder-2.7B reaching 25.07% exact match on Tufano Small (vs 23.86% baseline) and 16.19% on Tufano Medium (vs 15.36% baseline).",
       "evidence": "Table 1, Section 3.1.3. Results shown for 6 LLMs across 4 datasets with baseline comparisons from CodeXGLUE leaderboard.",
       "supported": "moderate"
     },
     {
       "claim": "Toggle with CodeParrot-110M generalizes to unseen Defects4J bugs, fixing 41 bugs at Top-10 (vs 36 for Recoder), 58 at Top-30 (vs 51), 64 at Top-50 (vs 62), and 74 at Top-100 (vs 70).",
       "evidence": "Table 3, Section 3.2.2. Comparison against 7 existing APR methods using test-case validation on 240 single-hunk Defects4J bugs.",
       "supported": "strong"
     },
     {
       "claim": "Prompt design dramatically impacts bug fixing accuracy: for CodeGPT-110M, accuracy improves from 16.07% (Prompt 1) to 56.98% (Prompt 4) with ground-truth locations.",
       "evidence": "Table 4, Section 3.3. Controlled experiment with 6 LLMs × 4 prompts on Tufano Small, using ground-truth locations to isolate prompt effect.",
       "supported": "strong"
     },
     {
       "claim": "Additional contextual information (buggy line numbers, code review comments) significantly improves bug localization accuracy, e.g., starting token accuracy on Tufano Medium increases from 26.73% to 56.66% with buggy line numbers.",
       "evidence": "Table 5, Section 3.4. Comparison of location prediction with/without context across 3 datasets.",
       "supported": "strong"
     },
     {
       "claim": "The adjustment module consistently improves bug fixing accuracy across all 16 dataset-model combinations tested.",
       "evidence": "Table 6, Section 3.5. All 16 cells show improvement, though magnitudes range from 0.17pp to 2.02pp.",
       "supported": "moderate"
     },
     {
       "claim": "Larger LLMs yield better bug fixing accuracy after fine-tuning, with larger models consistently outperforming smaller counterparts within the same family.",
       "evidence": "Table 1, Section 3.1.3. Within CodeGen (350M→2B) and PolyCoder (400M→2.7B), larger models outperform on all datasets.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No code or model release",
       "detail": "Toggle's implementation, fine-tuned models, and experimental scripts are not released, making independent verification impossible despite claims of SOTA performance."
     },
     {
       "flag": "No error bars or statistical tests",
       "detail": "All results are single point estimates with no confidence intervals, significance tests, or variance measures. Claims of outperformance are based on raw number comparisons (e.g., 25.07% vs 23.86%) that could be within noise."
     },
     {
       "flag": "Missing training hyperparameters",
       "detail": "No learning rate, batch size, epochs, optimizer, or generation parameters (temperature, beam size) are reported despite fine-tuning 6 models across multiple configurations."
     },
     {
       "flag": "Undisclosed corporate funding",
       "detail": "Five of eight authors are affiliated with Amazon Web Services but no funding source or competing interests statement is provided."
     },
     {
       "flag": "Incomplete baseline comparison for Defects4J",
       "detail": "The Top-200 and ≥Top-500 columns in Table 3 are blank for Toggle ('-'), preventing fair comparison against methods like TENURE (129 at ≥Top-500) and AlphaRepair (110) at higher patch budgets."
     }
   ],
   "cited_papers": [
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "relevance": "Explores LLMs for APR showing strong fixing capabilities, directly relevant to LLM-based code repair evaluation."
     },
     {
       "title": "Automated Program Repair in the Era of Large Pre-Trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Studies LLMs for APR with prompt engineering, a key baseline and related approach."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair zero-shot APR method, a primary baseline in the Defects4J evaluation."
     },
     {
       "title": "CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis",
       "authors": ["Erik Nijkamp", "Bo Pang", "Hiroaki Hayashi"],
       "year": 2023,
       "arxiv_id": "2203.13474",
       "relevance": "One of the primary LLM backbones used in Toggle's experiments."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Yue Wang", "Weishi Wang", "Shafiq R. Joty", "Steven C. H. Hoi"],
       "year": 2021,
       "relevance": "Encoder backbone used for Toggle's bug localization model, core to the framework."
     },
     {
       "title": "A Systematic Evaluation of Large Language Models of Code",
       "authors": ["Frank F. Xu", "Uri Alon", "Graham Neubig", "Vincent J. Hellendoorn"],
       "year": 2022,
       "arxiv_id": "2202.13169",
       "relevance": "PolyCoder evaluation paper; PolyCoder is a primary LLM backbone in the experiments."
     },
     {
       "title": "KNOD: Domain Knowledge Distilled Tree Decoder for Automated Program Repair",
       "authors": ["Nan Jiang", "Thibaud Lutellier", "Yiling Lou", "Lin Tan"],
       "year": 2023,
       "relevance": "Key APR baseline in Defects4J comparison, representing edit-based approaches."
     },
     {
       "title": "Tare: Type-Aware Neural Program Repair",
       "authors": ["Qihao Zhu", "Zeyu Sun", "Wenjie Zhang", "Yingfei Xiong", "Lu Zhang"],
       "year": 2023,
       "relevance": "Recent APR baseline in Defects4J evaluation, fixing 109 bugs at ≥Top-500."
     },
     {
       "title": "CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation",
       "authors": ["Shuai Lu", "Daya Guo", "Shuo Ren"],
       "year": 2021,
       "relevance": "Primary benchmark used for evaluation, widely used for code ML tasks."
     },
     {
       "title": "Defects4J: A Database of existing faults to enable controlled testing studies for Java programs",
       "authors": ["René Just", "Darioush Jalali", "Michael D. Ernst"],
       "year": 2014,
       "relevance": "Standard bug benchmark used for generalizability evaluation with test-case validation."
     },
     {
       "title": "An Empirical Study of Deep Learning Models for Vulnerability Detection",
       "authors": ["Benjamin Steenhoek", "Md Mahbubur Rahman", "Richard Jiles", "Wei Le"],
       "year": 2023,
       "relevance": "Empirical study of DL for vulnerability detection, relevant to LLM-based code analysis evaluation."
     },
     {
       "title": "How Effective Are Neural Networks for Fixing Security Vulnerabilities",
       "authors": ["Yi Wu", "Nan Jiang", "Hung Viet Pham"],
       "year": 2023,
       "relevance": "Evaluates neural networks for vulnerability fixing, closely related to LLM-based program repair."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Toggle addresses a practical problem (automated bug fixing) but no code is released, limiting immediate use."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Token-level vs line-level localization is a novel granularity shift, but does not fundamentally challenge conventional wisdom about LLM-based APR."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the work focuses on improving code repair accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward benchmark improvement paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released for anyone to try."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Amazon Web Services is well-known but not a top-tier AI research brand; the work is published at FSE, a respected but niche SE venue."
     }
   }
 }