ai-research-survey

scan.json (33234B)

---

 {
   "paper": {
     "title": "Fixing 7,400 Bugs for 1$: Cheap Crash-Site Program Repair",
     "authors": [
       "Han Zheng",
       "Ilia Shumailov",
       "Tianqi Fan",
       "Aiden Hall",
       "Mathias Payer"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.13103",
     "doi": "10.48550/arXiv.2505.13103"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "WILLIAMT, a template-guided crash-site repair tool, fixes 46.1% of 358 ARVO memory corruption bugs at $0.0026/bug (vs $0.93/bug for CodeRover-S). The combined WILLIAMT+CodeRover-S pipeline improves the fixing rate by 29.6% while reducing token cost by 45.9% compared to CodeRover-S alone. Non-reasoning models (Claude 3.5 Haiku) outperform reasoning models at the crash-site analysis task, and local models (Gemma3:27B) achieve 96.4% of GPT-4o's performance. Manual review reveals only 56 of 165 plausible patches (34%) are truly correct across diverse inputs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly states 'We promise to fully release WILLIAMT upon paper acceptance to support open science' (Section 1, repeated in Section 7). A promise of future release does not count as released code."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses ARVO, a publicly available benchmark (reference [30]). The paper states ARVO 'reconstructs and curates a reproducible dataset of OSS bugs specifically tailored for APR evaluation' and all bugs originate from OSS-Fuzz."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('Ubuntu 22.04 server equipped with AMD EPYC 7302P and 64GB RAM', 'RTX 4090 GPU', 'Mac Mini M4') but no software dependency specifications, requirements.txt, Dockerfile, or library versions are provided. The ARVO Docker images provide reproducibility for the benchmark, but the WILLIAMT tool's own environment is not specified."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Code is not yet released, making reproduction impossible."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., '54.5%', '46.1%', '$0.0026'). No confidence intervals, error bars, or uncertainty measures appear in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims like 'WILLIAMT reduces token cost by 45.9%' and 'increases the bug-fixing rate to 73.5%' are based solely on comparing raw numbers without any statistical significance test."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are given with baseline context throughout: '45.9% cost reduction', '29.6% improvement in fixing rate', '357 times more bugs per dollar', '99.7% token cost reduction while preserving over 86.7% performance' (Section 5.1-5.3). Absolute numbers and percentages are provided for both systems."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 358-bug sample is defined by selection criteria ('all HOF, SOF, UAF and GOF bugs that can be compiled within 15 minutes') but there is no justification for why this sample size is adequate for the claims being made, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. WILLIAMT is described as 'one-shot' (one trial, one patch), and no standard deviation, variance, or spread measures are reported across any experimental condition."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three SoTA baselines are compared: AutoCodeRover-S [60], Agentless [54], and VulMaster [64]. Results are presented in Figure 4 and the Venn diagram in Figure 5c."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2024: CodeRover-S (arXiv 2411.03346, 2024), Agentless (ISSTA 2024), and VulMaster (ICSE 2024). These represent the current state of the art in LLM-based APR."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "WILLIAMT has two main components (regex-based context retrieval and template-guided patch generation) but no ablation study removes individual components to measure their contribution. The multi-LLM comparison (Figure 6) varies the backend but not the system architecture."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: plausible fix rate, token cost (USD), execution time, CPU usage, and a detailed fix classification (Plausible, Multiple, Compiles, No Patch, No Code). Appendix A additionally reports actual fix ratio after manual review."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Figure 9) presents manual evaluation of all 165 plausible patches from WILLIAMT-GPT-4o: automated validation of execution consistency, then manual review of robustness across diverse inputs. 56 of 165 are found truly correct."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 358 bugs are used for evaluation. There is no separation into development and test sets, and no discussion of whether prompts or templates were tuned on any subset of the data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by fix outcome (Plausible/Multiple/Compiles/No Patch/No Code in Figures 4 and 6) and by bug type (HBO 52.3%, UAF 11.3%, SBO 8.9%, GBO 4.6% in Figure 11). The Venn diagram (Figure 5c) shows overlap across tools."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses three failure modes: incorrect crash site analysis (LLM returns inaccurate variables), semantically disruptive patch insertion (breaking control flow), and the imprecise plausible metric. Appendix A provides a detailed failure funnel (Figure 9): 165→95→56 through successive filtering."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reasoning models (DeepSeek-R1, o3-mini) are shown to cost 5.7-5.9× more than Claude35-Haiku 'without providing notable improvements over non-reasoning LLMs' (Section 5.3). Gemma3:1b fixes only 10 bugs. VulMaster 'resolves only 5 bugs in total.' The manual review shows only 56/165 plausible patches are truly correct."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the combined pipeline achieves '73.5% (+29.6%)' fixing rate, but 195 (CodeRover-S) + 60 additional = 255/358 = 71.2%, not 73.5%. More critically, the Conclusion (Section 7) swaps the two key numbers: 'reduces token usage by 29.6% and improves the fixing rate by 45.9%', contradicting the abstract's '45.9% cost reduction' and '29.6% fixing rate improvement.' These internal inconsistencies undermine the abstract claims."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims ('crash-site repair reduces cost while maintaining effectiveness') are supported by controlled comparison: same benchmark (358 ARVO bugs), same LLM (gpt-4o-2024-08-06) for the head-to-head, with WilliamT tested against CodeRover-S, Agentless, and VulMaster under equivalent conditions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Fixing 7,400 Bugs for 1$' is not supported anywhere in the paper body — the number 7,400 never appears in the text. The evaluation covers 358 bugs of 4 types (HBO, GBO, SBO, UAF) on one benchmark (ARVO). At the reported cost of $0.0026/bug, $1 covers ~385 bugs, not 7,400. The claim in the title is an unsupported extrapolation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses failure modes of WILLIAMT but does not consider alternative explanations for its performance. For example, there is no discussion of whether ARVO bugs are particularly amenable to crash-site repair, whether the specific bug types selected bias results, or whether LLMs may have seen the fixes in training."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Table 1, Figure 9) extensively distinguishes between the 'plausible fix' proxy metric and actual correctness. They show the plausible metric 'does not ensure that the program's functionality is preserved' and demonstrate that only 56/165 plausible fixes survive manual review. This is an unusually thorough proxy-outcome analysis."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-4o is specified with a snapshot date ('gpt-4o-2024-08-06'). All other models use marketing names without API versions or snapshot dates: 'DeepSeek (V3, R1)', 'Claude (3.5-Haiku, 3.7-Sonnet)', 'o3-mini', 'Gemma3 (27B, 12B, 4B, 1B)'. Per the schema, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 15 (Appendix C) provides the full prompt text used for crash site analysis, including the system instruction, variable format specifications, examples, and the <issue> tag structure. The patch templates are fully specified in Figures 12 and 13."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for any of the models tested. These settings significantly affect output and are not mentioned."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The WILLIAMT workflow is described in detail: Figure 1 shows the pipeline, Section 4 describes regex-based context retrieval and template-guided patch generation, Appendix B provides the vulnerability templates, and Appendix C details the regex parsing and prompt preparation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Bug selection criteria are stated: 'all HOF, SOF, UAF and GOF bugs (358 bugs) that can be compiled within 15 minutes following the recommended practice' from ARVO. Appendix C documents the regex-based preprocessing of AddressSanitizer reports including stack trace parsing, bug type classification, and code context extraction."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Discussion') serves as a dedicated limitations section, discussing three categories of limitations: incorrect crash site analysis, semantically disruptive patch insertion, and imprecise plausible metric. Each is discussed substantively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'the code snippet provided to the LLM may lack the correct variable due to limited context' (specific to their context window), ''No Patch' failures account for approximately 37% of all bugs, with many caused by such semantic violations' (quantified failure mode), and the plausible metric limitation with concrete data (only 56/165 are truly correct)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes to crash-site repair (not root cause), four memory corruption types (HBO, GBO, SBO, UAF), and acknowledges crash-site fixes are temporary: 'giving developers time to implement a permanent solution' (Section 3). Section 6 states the plausible metric 'may overestimate the number of truly correct fixes.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is released — no patch outputs, LLM responses, per-bug results, or detailed logs. Only aggregate results are presented. The ARVO benchmark is public but the experimental outputs are not."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: ARVO benchmark with 5,000+ bugs, filtered to 4 bug types (HBO, GBO, SBO, UAF) that compile within 15 minutes, yielding 358 bugs. Selection follows 'the recommended practice' from [60]."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is a standard public benchmark (ARVO)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from ARVO (5,000+ bugs) to 358 evaluation bugs lacks intermediate counts. How many HBO, GBO, SBO, UAF bugs exist in ARVO before the 15-minute compile filter? How many were excluded by the compile filter? These intermediate steps are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is disclosed anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Han Zheng and Mathias Payer at EPFL; Ilia Shumailov at Google DeepMind; Tianqi Fan at Google; Aiden Hall at Google."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of five authors are affiliated with Google/DeepMind. The evaluation relies on Google infrastructure (OSS-Fuzz, ClusterFuzz, ARVO). Google has a stake in demonstrating effective APR for OSS-Fuzz bugs. No funding disclosure makes independence impossible to assess."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no patent disclosures, and no financial interests declaration appears in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff date is stated for any of the LLMs used (GPT-4o, DeepSeek, Claude, Gemma3). The ARVO bugs come from publicly accessible OSS-Fuzz reports and open-source git histories that could appear in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen ARVO bug reports, crash traces, or ground-truth patches during training. The OSS-Fuzz reports and open-source commit histories containing the fixes are publicly available and likely in LLM training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ARVO bugs originate from OSS-Fuzz, with ground-truth fixes in public git repositories. These fixes were committed before most models' training cutoffs, creating significant contamination risk. This is not discussed or addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed per-bug inference costs are reported: $0.0026/bug for WILLIAMT vs $0.93/bug for CodeRover-S (Figure 5a). Figure 7 breaks down cost across all frontier LLMs, with DeepSeek-V3 at <0.03 cents/bug and Claude35-Haiku at ~0.09 cents/bug."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified (AMD EPYC 7302P, 64GB RAM, RTX 4090, Mac Mini M4). Execution times are reported: WILLIAMT <1 minute vs CodeRover-S ~43.5 minutes. Figure 8 provides detailed time breakdown for preprocessing and each LLM backend. Total cost for all 358 bugs: <$0.68 for WILLIAMT."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. WILLIAMT uses a one-shot design and all results appear to be single-run. LLM output stochasticity is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "WILLIAMT is explicitly described as one-shot: 'WILLIAMT performs a single trial and applies only one patch throughout the entire fixing pipeline' (Section 5.1). CodeRover-S is stated to use 'up to three trials per bug' with 'up to 18 attempts.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of how the prompt template, context window size (2 lines before/after), or patch templates were developed or tuned. No search budget reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the specific prompt design, template structure, or context window size were selected. The paper presents one configuration without discussing alternatives tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their own WILLIAMT against SoTA tools. While they import CodeRover-S results from its original paper for fairness, they do not acknowledge the general bias of evaluating one's own system or discuss how this might affect template/prompt design choices."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is explicitly analyzed as a function of cost: Figures 5a-5b compare cost and time between WILLIAMT and CodeRover-S, Figure 7 shows cost vs performance across LLM backends, and Section 5.3 discusses cost-performance tradeoffs for reasoning vs non-reasoning models."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes ARVO as a reliable benchmark and notes it 'ensures that all bugs are both ground-truth and reproducible' but does not question whether fixing 358 ARVO bugs of 4 types generalizes to real-world APR capability. No discussion of whether ARVO's specific bug distribution represents real-world vulnerability patterns."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing LLM backends (Figure 6), all models use the same WILLIAMT scaffold, isolating the model variable. The SoTA comparison (Figure 4) compares complete systems as intended — the scaffold IS the thing being tested."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. ARVO bugs have ground-truth fixes committed to public repositories before the LLMs' training cutoffs. Models may have learned the exact patches from training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. The crash-site analysis prompt (Figure 15) provides sanitizer output and source context, which is the intended input. However, no analysis of whether the LLM might be retrieving memorized fixes rather than reasoning about the crash site."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not discussed. Multiple bugs may come from the same project or share similar code patterns. No analysis of whether results are driven by a few projects or are independent across bugs."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference tests, temporal splits, or decontamination procedures."
       }
     }
   },
   "claims": [
     {
       "claim": "WILLIAMT saves 97.7% CPU usage and 99.7% token cost while preserving over 86.7% of CodeRover-S performance.",
       "evidence": "Section 5.1, Figures 5a-5b: WILLIAMT costs $0.0026/bug vs CodeRover-S $0.93/bug. WILLIAMT fixes 165/358 bugs (46.1%) vs CodeRover-S 195/358 (54.5%), giving 84.6% relative performance. Execution time: 58.7s vs 2610.0s.",
       "supported": "moderate"
     },
     {
       "claim": "Combined WILLIAMT+CodeRover-S pipeline reduces token cost by 45.9% and improves fixing rate by 29.6% compared to CodeRover-S alone.",
       "evidence": "Section 5.2, Figure 5c Venn diagram. The pipeline achieves 60 additional plausible fixes. However, the Conclusion (Section 7) swaps these numbers ('reduces token usage by 29.6% and improves the fixing rate by 45.9%'), and the abstract claim of 73.5% combined rate (263/358) doesn't align with 195+60=255/358=71.2%.",
       "supported": "weak"
     },
     {
       "claim": "Claude 3.5 Haiku achieves the highest fixing rate at 47.5% among all tested models.",
       "evidence": "Section 5.3, Figure 6: Claude35-haiku fixes 170 bugs (47.5%), followed by Claude37-sonnet (166, 46.4%) and GPT-4o (165, 46.1%). Single run without error bars.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning models do not improve fixing performance despite substantially higher costs.",
       "evidence": "Section 5.3, Figure 7: DeepSeek-R1 costs 0.70 cents/bug and o3-mini 0.72 cents/bug (5.7-5.9× more than Claude35-Haiku at 0.09 cents) with 159 and 153 fixes respectively, compared to Claude35-Haiku's 170. Reasoning models generate much more output tokens without better results.",
       "supported": "moderate"
     },
     {
       "claim": "Gemma3:27B running locally achieves 96.4% of GPT-4o's performance.",
       "evidence": "Section 5.3, Figure 6: Gemma3:27B fixes 163 bugs vs GPT-4o's 165. 163/165 = 98.8% (paper claims 96.4%, possibly calculated differently). Single benchmark, one run, no error bars.",
       "supported": "moderate"
     },
     {
       "claim": "Only 56 of 165 plausible WILLIAMT patches (34%) are truly correct across diverse inputs.",
       "evidence": "Appendix A, Figure 9: Detailed manual review pipeline shows 165 plausible → 95 no early exit → 56 correct for broader inputs. This honest assessment significantly deflates the headline plausible fix rate.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Unexplained title number",
       "detail": "The title claims 'Fixing 7,400 Bugs for 1$' but the number 7,400 never appears in the paper body. The evaluation covers only 358 bugs. At the reported cost of $0.0026/bug, $1 covers ~385 bugs. Even with the cheapest model (DeepSeek-V3 at ~$0.0003/bug), $1 covers ~3,333 bugs. The title number appears to be unsupported."
     },
     {
       "flag": "Internal numerical inconsistency",
       "detail": "The Conclusion (Section 7) states 'reduces token usage by 29.6% and improves the fixing rate by 45.9%', directly swapping the two key numbers from the abstract and Section 5.2 ('reduces token cost by 45.9%' and '29.6% improvement in fixing rate'). The abstract claims 73.5% combined fixing rate but 195+60=255/358=71.2%."
     },
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results are single-run point estimates. LLM outputs are stochastic, and the one-shot design means a single different LLM response could change any individual bug's outcome. No uncertainty quantification is provided."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The ARVO benchmark draws from OSS-Fuzz, with ground-truth fixes committed to public GitHub repositories. All LLMs tested likely encountered these bug reports and patches during training. The paper does not discuss this contamination risk at all."
     },
     {
       "flag": "Google affiliation conflict of interest",
       "detail": "Three of five authors are affiliated with Google/DeepMind. The evaluation relies on Google infrastructure (OSS-Fuzz, ClusterFuzz, ARVO). Google's Gemma3 model is shown to perform well. No funding disclosure or conflict-of-interest statement is provided."
     },
     {
       "flag": "Plausible metric inflates success",
       "detail": "The headline results use the 'plausible fix' metric (PoC doesn't crash). The paper's own manual review (Appendix A) shows only 56/165 (34%) of plausible patches are truly correct, meaning the actual fixing rate is closer to 15.6% (56/358), not the reported 46.1%. This affects all compared tools equally, but the authors deserve credit for reporting the gap."
     }
   ],
   "cited_papers": [
     {
       "title": "AutoCodeRover: Autonomous program improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Primary APR baseline using LLM agents; demonstrates agent-based program repair on real bugs."
     },
     {
       "title": "Fixing security vulnerabilities with AI in OSS-Fuzz",
       "authors": ["Yuntong Zhang", "Jiawei Wang", "Dominic Berzin", "Martin Mirchev", "Dongge Liu", "Abhishek Arya", "Oliver Chang", "Abhik Roychoudhury"],
       "year": 2024,
       "arxiv_id": "2411.03346",
       "relevance": "CodeRover-S, the top-performing baseline; evaluates AI-based vulnerability repair on the same ARVO benchmark."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Agentless APR framework; demonstrates conversational LLM-based program repair with cost analysis."
     },
     {
       "title": "Out of sight, out of mind: Better automatic vulnerability repair by broadening input ranges and sources",
       "authors": ["Xin Zhou", "Kisub Kim", "Bowen Xu", "DongGyun Han", "David Lo"],
       "year": 2024,
       "relevance": "VulMaster baseline; APR tool for vulnerability repair that broadens input context."
     },
     {
       "title": "ARVO: Atlas of reproducible vulnerabilities for open source software",
       "authors": ["Xiang Mei", "Pulkit Singh Singaria", "Jordi Del Castillo"],
       "year": 2024,
       "arxiv_id": "2408.02153",
       "relevance": "The primary evaluation benchmark providing 5,000+ ground-truth reproducible memory corruption bugs from OSS-Fuzz."
     },
     {
       "title": "Template-guided program repair in the era of large language models",
       "authors": ["Kai Huang", "Jian Zhang", "Xiangxin Meng", "Yang Liu"],
       "year": 2025,
       "relevance": "Template-guided APR using LLMs; directly related approach to WILLIAMT's template-based method."
     },
     {
       "title": "Code repair with LLMs gives an exploration-exploitation tradeoff",
       "authors": ["Hao Tang", "Keya Hu", "Jin Zhou", "Si Cheng Zhong", "Wei-Long Zheng", "Xujie Si", "Kevin Ellis"],
       "year": 2024,
       "relevance": "Analyzes the exploration-exploitation tradeoff in LLM-based code repair, relevant to understanding multi-attempt vs one-shot strategies."
     },
     {
       "title": "An empirical study on fine-tuning large language models of code for automated program repair",
       "authors": ["Kai Huang", "Xiangxin Meng", "Jian Zhang", "Yang Liu", "Wenjie Wang", "Shuhao Li", "Yuqing Zhang"],
       "year": 2023,
       "relevance": "Empirical evaluation of fine-tuning LLMs for APR; defines the Plausible/Multiple/Compiles/No Patch repair classification used here."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Zero-shot LLM-based APR demonstrating that LLMs can repair code without task-specific training."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning LLM used as a baseline; shown to produce excessive output tokens without improving repair performance."
     },
     {
       "title": "Sequencer: Sequence-to-sequence learning for end-to-end program repair",
       "authors": ["Zimin Chen", "Steve Kommrusch", "Michele Tufano"],
       "year": 2019,
       "relevance": "Early neural program repair approach using sequence-to-sequence learning."
     },
     {
       "title": "An empirical study on learning bug-fixing patches in the wild via neural machine translation",
       "authors": ["Michele Tufano", "Cody Watson", "Gabriele Bavota"],
       "year": 2019,
       "relevance": "Neural machine translation approach to learning bug-fixing patches from real-world data."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses a real developer pain point (fuzzer bug backlog), designed for local deployment on consumer hardware (Mac Mini M4), and shows concrete cost savings."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that root-cause analysis and expensive frontier models are necessary for effective program repair; shows crash-site patches can be useful."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses security vulnerabilities (memory corruption) but proposes mitigations rather than raising new concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly argues expensive agent-based APR approaches are overkill for many bugs, but framed constructively as complementary rather than adversarial."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code is not released; promised only upon acceptance. No live demo or installable tool available."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Three authors from Google/Google DeepMind, two from EPFL — recognizable institutions but not about a flagship Google product."
     }
   }
 }