ai-research-survey

scan-v4.json (37511B)

---

 {
   "scan_version": 4,
   "paper_type": "empirical",
   "paper": {
     "title": "Fixing 7,400 Bugs for 1$: Cheap Crash-Site Program Repair",
     "authors": [
       "Han Zheng",
       "Ilia Shumailov",
       "Tianqi Fan",
       "Aiden Hall",
       "Mathias Payer"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2505.13103",
     "doi": "10.48550/arXiv.2505.13103"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims the combined pipeline achieves '73.5% (+29.6%)' fixing rate, but 195 (CodeRover-S) + 60 additional = 255/358 = 71.2%, not 73.5%. More critically, the Conclusion (Section 7) swaps the two key numbers: 'reduces token usage by 29.6% and improves the fixing rate by 45.9%', contradicting the abstract's '45.9% cost reduction' and '29.6% fixing rate improvement.' These internal inconsistencies undermine the abstract claims.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claims ('crash-site repair reduces cost while maintaining effectiveness') are supported by controlled comparison: same benchmark (358 ARVO bugs), same LLM (gpt-4o-2024-08-06) for the head-to-head, with WilliamT tested against CodeRover-S, Agentless, and VulMaster under equivalent conditions.",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Fixing 7,400 Bugs for 1$' is not supported anywhere in the paper body — the number 7,400 never appears in the text. The evaluation covers 358 bugs of 4 types (HBO, GBO, SBO, UAF) on one benchmark (ARVO). At the reported cost of $0.0026/bug, $1 covers ~385 bugs, not 7,400. The claim in the title is an unsupported extrapolation.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6 discusses failure modes of WILLIAMT but does not consider alternative explanations for its performance. For example, there is no discussion of whether ARVO bugs are particularly amenable to crash-site repair, whether the specific bug types selected bias results, or whether LLMs may have seen the fixes in training.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A (Table 1, Figure 9) extensively distinguishes between the 'plausible fix' proxy metric and actual correctness. They show the plausible metric 'does not ensure that the program's functionality is preserved' and demonstrate that only 56/165 plausible fixes survive manual review. This is an unusually thorough proxy-outcome analysis.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 ('Discussion') serves as a dedicated limitations section, discussing three categories of limitations: incorrect crash site analysis, semantically disruptive patch insertion, and imprecise plausible metric. Each is discussed substantively.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats are specific to this study: 'the code snippet provided to the LLM may lack the correct variable due to limited context' (specific to their context window), ''No Patch' failures account for approximately 37% of all bugs, with many caused by such semantic violations' (quantified failure mode), and the plausible metric limitation with concrete data (only 56/165 are truly correct).",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly scopes to crash-site repair (not root cause), four memory corruption types (HBO, GBO, SBO, UAF), and acknowledges crash-site fixes are temporary: 'giving developers time to implement a permanent solution' (Section 3). Section 6 states the plausible metric 'may overestimate the number of truly correct fixes.'",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source, acknowledgments section, or grant information is disclosed anywhere in the paper.",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Han Zheng and Mathias Payer at EPFL; Ilia Shumailov at Google DeepMind; Tianqi Fan at Google; Aiden Hall at Google.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Three of five authors are affiliated with Google/DeepMind. The evaluation relies on Google infrastructure (OSS-Fuzz, ClusterFuzz, ARVO). Google has a stake in demonstrating effective APR for OSS-Fuzz bugs. No funding disclosure makes independence impossible to assess.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement, no patent disclosures, and no financial interests declaration appears in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined precisely: 'crash-site repair' vs. root-cause repair is explained with an example (Figure 3), 'plausible fix' is defined operationally (patched program does not crash on PoC), and memory corruption categories (spatial/temporal) are defined in Section 2.2.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Section 1 ends with a four-bullet contribution list explicitly identifying: crash-site repair proposal, template-guided patch generation technique, WILLIAMT implementation/evaluation, and open-source promise.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 situates the work relative to existing APR agents (AutoCodeRover-S, Agentless, VulMaster) with direct comparisons in the evaluation; the paper contrasts its crash-site approach with root-cause-oriented methods and explains why root-cause analysis fails in practice (Figure 3 example).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "The paper explicitly states 'We promise to fully release WILLIAMT upon paper acceptance to support open science' (Section 1, repeated in Section 7). A promise of future release does not count as released code.",
           "source": "opus"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The evaluation uses ARVO, a publicly available benchmark (reference [30]). The paper states ARVO 'reconstructs and curates a reproducible dataset of OSS bugs specifically tailored for APR evaluation' and all bugs originate from OSS-Fuzz.",
           "source": "opus"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Hardware is mentioned ('Ubuntu 22.04 server equipped with AMD EPYC 7302P and 64GB RAM', 'RTX 4090 GPU', 'Mac Mini M4') but no software dependency specifications, requirements.txt, Dockerfile, or library versions are provided. The ARVO Docker images provide reproducibility for the benchmark, but the WILLIAMT tool's own environment is not specified.",
           "source": "opus"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Code is not yet released, making reproduction impossible.",
           "source": "opus"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results are reported as single point estimates (e.g., '54.5%', '46.1%', '$0.0026'). No confidence intervals, error bars, or uncertainty measures appear in any table or figure.",
           "source": "opus"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims like 'WILLIAMT reduces token cost by 45.9%' and 'increases the bug-fixing rate to 73.5%' are based solely on comparing raw numbers without any statistical significance test.",
           "source": "opus"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Effect sizes are given with baseline context throughout: '45.9% cost reduction', '29.6% improvement in fixing rate', '357 times more bugs per dollar', '99.7% token cost reduction while preserving over 86.7% performance' (Section 5.1-5.3). Absolute numbers and percentages are provided for both systems.",
           "source": "opus"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 358-bug sample is defined by selection criteria ('all HOF, SOF, UAF and GOF bugs that can be compiled within 15 minutes') but there is no justification for why this sample size is adequate for the claims being made, nor any power analysis.",
           "source": "opus"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "All results appear to be from single runs. WILLIAMT is described as 'one-shot' (one trial, one patch), and no standard deviation, variance, or spread measures are reported across any experimental condition.",
           "source": "opus"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Three SoTA baselines are compared: AutoCodeRover-S [60], Agentless [54], and VulMaster [64]. Results are presented in Figure 4 and the Venn diagram in Figure 5c.",
           "source": "opus"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "All baselines are from 2024: CodeRover-S (arXiv 2411.03346, 2024), Agentless (ISSTA 2024), and VulMaster (ICSE 2024). These represent the current state of the art in LLM-based APR.",
           "source": "opus"
         },
         "ablation_study": {
           "applies": true,
           "answer": false,
           "justification": "WILLIAMT has two main components (regex-based context retrieval and template-guided patch generation) but no ablation study removes individual components to measure their contribution. The multi-LLM comparison (Figure 6) varies the backend but not the system architecture.",
           "source": "opus"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Multiple metrics are reported: plausible fix rate, token cost (USD), execution time, CPU usage, and a detailed fix classification (Plausible, Multiple, Compiles, No Patch, No Code). Appendix A additionally reports actual fix ratio after manual review.",
           "source": "opus"
         },
         "human_evaluation": {
           "applies": true,
           "answer": true,
           "justification": "Appendix A (Figure 9) presents manual evaluation of all 165 plausible patches from WILLIAMT-GPT-4o: automated validation of execution consistency, then manual review of robustness across diverse inputs. 56 of 165 are found truly correct.",
           "source": "opus"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": false,
           "justification": "All 358 bugs are used for evaluation. There is no separation into development and test sets, and no discussion of whether prompts or templates were tuned on any subset of the data.",
           "source": "opus"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Results are broken down by fix outcome (Plausible/Multiple/Compiles/No Patch/No Code in Figures 4 and 6) and by bug type (HBO 52.3%, UAF 11.3%, SBO 8.9%, GBO 4.6% in Figure 11). The Venn diagram (Figure 5c) shows overlap across tools.",
           "source": "opus"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Section 6 discusses three failure modes: incorrect crash site analysis (LLM returns inaccurate variables), semantically disruptive patch insertion (breaking control flow), and the imprecise plausible metric. Appendix A provides a detailed failure funnel (Figure 9): 165→95→56 through successive filtering.",
           "source": "opus"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Reasoning models (DeepSeek-R1, o3-mini) are shown to cost 5.7-5.9× more than Claude35-Haiku 'without providing notable improvements over non-reasoning LLMs' (Section 5.3). Gemma3:1b fixes only 10 bugs. VulMaster 'resolves only 5 bugs in total.' The manual review shows only 56/165 plausible patches are truly correct.",
           "source": "opus"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Only GPT-4o is specified with a snapshot date ('gpt-4o-2024-08-06'). All other models use marketing names without API versions or snapshot dates: 'DeepSeek (V3, R1)', 'Claude (3.5-Haiku, 3.7-Sonnet)', 'o3-mini', 'Gemma3 (27B, 12B, 4B, 1B)'. Per the schema, marketing names without snapshot dates do not count.",
           "source": "opus"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Figure 15 (Appendix C) provides the full prompt text used for crash site analysis, including the system instruction, variable format specifications, examples, and the <issue> tag structure. The patch templates are fully specified in Figures 12 and 13.",
           "source": "opus"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "No LLM hyperparameters (temperature, top-p, max tokens) are reported for any of the models tested. These settings significantly affect output and are not mentioned.",
           "source": "opus"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The WILLIAMT workflow is described in detail: Figure 1 shows the pipeline, Section 4 describes regex-based context retrieval and template-guided patch generation, Appendix B provides the vulnerability templates, and Appendix C details the regex parsing and prompt preparation.",
           "source": "opus"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Bug selection criteria are stated: 'all HOF, SOF, UAF and GOF bugs (358 bugs) that can be compiled within 15 minutes following the recommended practice' from ARVO. Appendix C documents the regex-based preprocessing of AddressSanitizer reports including stack trace parsing, bug type classification, and code context extraction.",
           "source": "opus"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "No raw data is released — no patch outputs, LLM responses, per-bug results, or detailed logs. Only aggregate results are presented. The ARVO benchmark is public but the experimental outputs are not.",
           "source": "opus"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Data collection is described: ARVO benchmark with 5,000+ bugs, filtered to 4 bug types (HBO, GBO, SBO, UAF) that compile within 15 minutes, yielding 358 bugs. Selection follows 'the recommended practice' from [60].",
           "source": "opus"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants. The data source is a standard public benchmark (ARVO).",
           "source": "opus"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": false,
           "justification": "The pipeline from ARVO (5,000+ bugs) to 358 evaluation bugs lacks intermediate counts. How many HBO, GBO, SBO, UAF bugs exist in ARVO before the 15-minute compile filter? How many were excluded by the compile filter? These intermediate steps are not documented.",
           "source": "opus"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training data cutoff date is stated for any of the LLMs used (GPT-4o, DeepSeek, Claude, Gemma3). The ARVO bugs come from publicly accessible OSS-Fuzz reports and open-source git histories that could appear in training data.",
           "source": "opus"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether the LLMs may have seen ARVO bug reports, crash traces, or ground-truth patches during training. The OSS-Fuzz reports and open-source commit histories containing the fixes are publicly available and likely in LLM training corpora.",
           "source": "opus"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "ARVO bugs originate from OSS-Fuzz, with ground-truth fixes in public git repositories. These fixes were committed before most models' training cutoffs, creating significant contamination risk. This is not discussed or addressed.",
           "source": "opus"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "opus"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Detailed per-bug inference costs are reported: $0.0026/bug for WILLIAMT vs $0.93/bug for CodeRover-S (Figure 5a). Figure 7 breaks down cost across all frontier LLMs, with DeepSeek-V3 at <0.03 cents/bug and Claude35-Haiku at ~0.09 cents/bug.",
           "source": "opus"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": true,
           "justification": "Hardware is specified (AMD EPYC 7302P, 64GB RAM, RTX 4090, Mac Mini M4). Execution times are reported: WILLIAMT <1 minute vs CodeRover-S ~43.5 minutes. Figure 8 provides detailed time breakdown for preprocessing and each LLM backend. Total cost for all 358 bugs: <$0.68 for WILLIAMT.",
           "source": "opus"
         }
       },
       "experimental_rigor": {
         "seed_sensitivity_reported": {
           "applies": true,
           "answer": false,
           "justification": "No seed sensitivity analysis. WILLIAMT uses a one-shot design and all results appear to be single-run. LLM output stochasticity is not addressed.",
           "source": "opus"
         },
         "number_of_runs_stated": {
           "applies": true,
           "answer": true,
           "justification": "WILLIAMT is explicitly described as one-shot: 'WILLIAMT performs a single trial and applies only one patch throughout the entire fixing pipeline' (Section 5.1). CodeRover-S is stated to use 'up to three trials per bug' with 'up to 18 attempts.'",
           "source": "opus"
         },
         "hyperparameter_search_budget": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of how the prompt template, context window size (2 lines before/after), or patch templates were developed or tuned. No search budget reported.",
           "source": "opus"
         },
         "best_config_selection_justified": {
           "applies": true,
           "answer": false,
           "justification": "No explanation of how the specific prompt design, template structure, or context window size were selected. The paper presents one configuration without discussing alternatives tried.",
           "source": "opus"
         },
         "multiple_comparison_correction": {
           "applies": false,
           "answer": false,
           "justification": "No statistical tests are performed, so multiple comparison correction is not applicable.",
           "source": "opus"
         },
         "self_comparison_bias_addressed": {
           "applies": true,
           "answer": false,
           "justification": "The authors compare their own WILLIAMT against SoTA tools. While they import CodeRover-S results from its original paper for fairness, they do not acknowledge the general bias of evaluating one's own system or discuss how this might affect template/prompt design choices.",
           "source": "opus"
         },
         "compute_budget_vs_performance": {
           "applies": true,
           "answer": true,
           "justification": "Performance is explicitly analyzed as a function of cost: Figures 5a-5b compare cost and time between WILLIAMT and CodeRover-S, Figure 7 shows cost vs performance across LLM backends, and Section 5.3 discusses cost-performance tradeoffs for reasoning vs non-reasoning models.",
           "source": "opus"
         },
         "benchmark_construct_validity": {
           "applies": true,
           "answer": false,
           "justification": "The paper describes ARVO as a reliable benchmark and notes it 'ensures that all bugs are both ground-truth and reproducible' but does not question whether fixing 358 ARVO bugs of 4 types generalizes to real-world APR capability. No discussion of whether ARVO's specific bug distribution represents real-world vulnerability patterns.",
           "source": "opus"
         },
         "scaffold_confound_addressed": {
           "applies": true,
           "answer": true,
           "justification": "When comparing LLM backends (Figure 6), all models use the same WILLIAMT scaffold, isolating the model variable. The SoTA comparison (Figure 4) compares complete systems as intended — the scaffold IS the thing being tested.",
           "source": "opus"
         }
       },
       "data_leakage": {
         "temporal_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. ARVO bugs have ground-truth fixes committed to public repositories before the LLMs' training cutoffs. Models may have learned the exact patches from training data.",
           "source": "opus"
         },
         "feature_leakage_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. The crash-site analysis prompt (Figure 15) provides sanitizer output and source context, which is the intended input. However, no analysis of whether the LLM might be retrieving memorized fixes rather than reasoning about the crash site.",
           "source": "opus"
         },
         "non_independence_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Not discussed. Multiple bugs may come from the same project or share similar code patterns. No analysis of whether results are driven by a few projects or are independent across bugs.",
           "source": "opus"
         },
         "leakage_detection_method": {
           "applies": true,
           "answer": false,
           "justification": "No leakage detection or prevention method is used. No canary strings, membership inference tests, temporal splits, or decontamination procedures.",
           "source": "opus"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "WILLIAMT reduces token cost by 99.7% compared to CodeRover-S while retaining over 86.7% of CodeRover-S's bug-fixing performance",
       "evidence": "WILLIAMT uses GPT-4o and achieves 46.1% plausible fixing rate vs CodeRover-S's 54.5%, at $0.0026/bug vs $0.93/bug — a 357x cost reduction (Section 5.1, Figure 5a)",
       "supported": "strong"
     },
     {
       "claim": "Combining WILLIAMT with CodeRover-S achieves 29.6% more bugs fixed and 45.9% cost reduction compared to CodeRover-S alone",
       "evidence": "The combined pipeline fixes 60 additional plausible bugs (73.5% total) while reducing cost by 45.9%; supported by Section 5.2 and Figure 5c Venn diagram showing largely disjoint fix sets",
       "supported": "strong"
     },
     {
       "claim": "Non-reasoning models outperform reasoning models on crash-site repair due to template guidance eliminating the need for heavy reasoning",
       "evidence": "Claude35-Haiku (non-reasoning) achieves the highest fixing rate at 47.5%; DeepSeek-R1 and GPT-o3-mini are 5.7x–5.9x more expensive without better performance (Section 5.3, Figure 6–7)",
       "supported": "moderate"
     },
     {
       "claim": "Local LLMs running on consumer hardware can achieve reasonable APR performance: Gemma3:27B reaches 96.4% of GPT-4o's fixing performance on an RTX 4090 or Mac Mini M4",
       "evidence": "Figure 6 shows Gemma3:27B fixes 163/358 bugs vs GPT-4o's 165/358; Mac Mini M4 test with Gemma3:4b 'performs on par to RTX 4090' (Section 5.3)",
       "supported": "moderate"
     },
     {
       "claim": "The standard 'plausible fix' metric overstates repair quality: only 34% of plausible patches (56/165) are genuinely correct across a broader range of inputs",
       "evidence": "Appendix A: of 165 plausible patches, 95 avoid early exit on PoC, 56 are manually verified correct for other inputs; 39 block valid inputs (Figure 9)",
       "supported": "strong"
     },
     {
       "claim": "WILLIAMT and CodeRover-S fix largely disjoint sets of bugs, indicating complementary rather than overlapping capabilities",
       "evidence": "Figure 5c: WILLIAMT fixes 50 unique bugs, CodeRover-S fixes 52 unique bugs, with only 46+15 in the overlap region; described in Section 5.2",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "WILLIAMT achieves 46.1% plausible bug-fixing rate on 358 ARVO memory-corruption bugs at $0.0026/bug — a 357x cost reduction vs CodeRover-S ($0.93/bug) while retaining 86.7% of its performance. Combining WILLIAMT (first pass) with CodeRover-S (second pass on failures) achieves 73.5% overall fixing rate with 45.9% cost reduction, exploiting the largely complementary fix sets of the two approaches. Template guidance reduces LLM reasoning requirements: non-reasoning Claude35-Haiku outperforms all reasoning models on this task, and local Gemma3:27B achieves 96.4% of GPT-4o performance. However, manual review reveals that the standard plausible-fix metric substantially overstates actual repair quality: only 56/165 plausible patches (34%) are genuinely correct across inputs beyond the original PoC, yielding a true repair rate of ~15.6%.",
   "red_flags": [
     {
       "flag": "Title inflation / misleading cost claim",
       "detail": "The title '7,400 bugs for $1' derives from the cheapest possible configuration (DeepSeek-V3 at ~0.0135 cent/bug), not the primary GPT-4o evaluation ($0.0026/bug ≈ 384 bugs per dollar); the headline claim is the best-case, not the representative case."
     },
     {
       "flag": "Plausible metric greatly overstates correctness",
       "detail": "Manual review of WILLIAMT-GPT-4o shows only 56/165 plausible patches (34%) are correct for inputs beyond the PoC — yielding a true repair rate of ~15.6% vs the reported 46.1% plausible rate. This analysis was not performed for CodeRover-S or other baselines, making comparisons asymmetric."
     },
     {
       "flag": "Code not released",
       "detail": "WILLIAMT source code is promised only 'upon paper acceptance'; reproduction is not possible from the paper alone."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "ARVO bugs are real-world vulnerabilities from public OSS-Fuzz with published fixes in public repositories; LLMs may have been trained on the exact bug-fix commits, inflating all models' performance. This risk is never mentioned."
     },
     {
       "flag": "No ablation of system components",
       "detail": "The paper attributes cost reduction to both regex-based retrieval AND template-guided generation, but no ablation tests either component in isolation; individual contribution of each mechanism is unknown."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims (WILLIAMT vs. baselines, LLM-to-LLM comparisons) are made without significance tests; differences of a few percentage points between LLMs (e.g., 46.1% vs 47.5%) are treated as meaningful without uncertainty quantification."
     },
     {
       "flag": "Google conflict of interest undisclosed",
       "detail": "Two co-authors are from Google/Google DeepMind; the evaluation extensively benchmarks Google's Gemma3 series with favorable results; no competing interests disclosure appears."
     },
     {
       "flag": "No per-bug-category fixing rate breakdown",
       "detail": "Bug type distribution is shown (HBO 52.3%, UAF 11.3%, etc.) but fixing rates are never broken down by category, making it impossible to assess whether crash-site repair works uniformly or only for specific vulnerability types."
     }
   ],
   "cited_papers": [
     {
       "title": "Fixing Security Vulnerabilities with AI in OSS-Fuzz (AutoCodeRover-S)",
       "relevance": "Primary baseline; the paper directly compares to and combines with CodeRover-S, making this the central comparison point for APR performance claims"
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 each using ChatGPT (Agentless)",
       "relevance": "Second primary baseline and directly comparable cost-efficiency claim in the same domain"
     },
     {
       "title": "ARVO: Atlas of Reproducible Vulnerabilities for Open Source Software",
       "relevance": "The benchmark used for all evaluation; critical for understanding the dataset's scope and reproducibility guarantees"
     },
     {
       "title": "Out of Sight, Out of Mind: Better Automatic Vulnerability Repair by Broadening Input Ranges and Sources (VulMaster)",
       "relevance": "Third baseline system evaluated; performs poorly (5 plausible fixes), providing contrast for WILLIAMT's approach"
     },
     {
       "title": "Template-guided Program Repair in the Era of Large Language Models",
       "relevance": "Prior work on template-guided APR that motivates WILLIAMT's approach"
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "relevance": "Earlier version of primary baseline; understanding the progression from AutoCodeRover to CodeRover-S is important for evaluating the competitive landscape"
     },
     {
       "title": "AddressSanitizer: A Fast Address Sanity Checker",
       "relevance": "Core infrastructure: WILLIAMT relies on ASan reports as input; understanding ASan capabilities determines scope of crash-site repair approach"
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "relevance": "Establishes zero-shot LLM-based APR paradigm that WILLIAMT builds upon"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses a real developer pain point (fuzzer bug backlog), designed for local deployment on consumer hardware (Mac Mini M4), and shows concrete cost savings."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that root-cause analysis and expensive frontier models are necessary for effective program repair; shows crash-site patches can be useful."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Addresses security vulnerabilities (memory corruption) but proposes mitigations rather than raising new concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly argues expensive agent-based APR approaches are overkill for many bugs, but framed constructively as complementary rather than adversarial."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code is not released; promised only upon acceptance. No live demo or installable tool available."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Three authors from Google/Google DeepMind, two from EPFL — recognizable institutions but not about a flagship Google product."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45444062",
         "title": "Machine Learnability as a Measure of Order in Aperiodic Sequences",
         "points": 48,
         "comments": 5,
         "url": "https://news.ycombinator.com/item?id=45444062"
       },
       {
         "hn_id": "46697408",
         "title": "WildCAT3D: Appearance-Aware Multi-View Diffusion in the Wild",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46697408"
       },
       {
         "hn_id": "43401539",
         "title": "CriteoPrivateAd: RealWorld Bidding Dataset to Design Private Advertising Systems",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43401539"
       },
       {
         "hn_id": "43516923",
         "title": "UniHOPE: A Unified Approach for Hand-Only and Hand-Object Pose Estimation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43516923"
       },
       {
         "hn_id": "43496516",
         "title": "UniHOPE: A Unified Approach for Hand-Only and Hand-Object Pose Estimation",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43496516"
       },
       {
         "hn_id": "36016970",
         "title": "Visual Question Answering: Techniques and Common Trends in Recent Literature",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=36016970"
       },
       {
         "hn_id": "44686218",
         "title": "The Heteronomy of Algorithms",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=44686218"
       },
       {
         "hn_id": "47380252",
         "title": "Show HN: Karpathy's Autoresearch with Evolutionary Database",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=47380252"
       },
       {
         "hn_id": "40515506",
         "title": "Evaluating AI-Generated Code for C++, Fortran, Go, Java, Julia, Matlab, etc.",
         "points": 1,
         "comments": 2,
         "url": "https://news.ycombinator.com/item?id=40515506"
       },
       {
         "hn_id": "43104988",
         "title": "Aide: AI-Driven Exploration in the Space of Code (Arxiv)",
         "points": 1,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=43104988"
       }
     ],
     "top_points": 48,
     "total_points": 63,
     "total_comments": 9
   }
 }