ai-research-survey

scan.json (31253B)

---

 {
   "paper": {
     "title": "Rethinking Kernel Program Repair: Benchmarking and Enhancing LLMs with RGym",
     "authors": [
       "Kareem Shehada",
       "Yifan Wu",
       "Wyatt D. Feng",
       "Adithya Iyer",
       "Gryphon Kumfert",
       "Yangruibo Ding",
       "Zhiyun Qian"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025 Workshop: Evaluating the Evolving LLM Lifecycle",
     "arxiv_id": "2511.15757",
     "doi": "10.48550/arXiv.2511.15757"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RGym is a lightweight, platform-agnostic framework for evaluating LLM-based automated program repair on Linux kernel KASAN bugs, designed to run on local commodity hardware instead of requiring GCP. Using practical localization (bug-inducing commits and call stacks) instead of oracle guidance, the simple APR pipeline achieves 37.76% (GPT-4o with feedback) to 43.36% (GPT-5 Thinking) pass rates at under $0.20 per bug on 143 verified bugs. Manual verification of 31 patches reveals only 32% are plausibly correct while 45% are wrong despite passing the crash test, highlighting the unreliability of crash-based patch evaluation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. RGym is described but no release link is given."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The dataset of 143 KASAN bugs is described as organized from Syzbot 'into an easily consumable format' but no download link or data archive is provided. While Syzbot itself is public, the curated 143-bug subset is not released."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (56 cores, 160GB RAM, 1TB SSD) and Docker/QEMU are mentioned, but no requirements.txt, Dockerfile, conda environment, or detailed dependency list is provided. Not enough detail to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or reproduction scripts are provided. The paper describes the architecture at a high level but does not give commands or procedures to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Table 1 are point estimates (e.g., '37.76%', '43.36%') with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'SimpleAgent+Feedback achieves 37.76%' vs 'kGym-oracle achieves 2.8%' are made by comparing raw numbers with no statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Improvements are reported with baseline context throughout: 'a 6.99% improvement over kGym's oracle-guided solution,' 'bad patches are reduced by 76%,' and costs are compared in absolute terms (e.g., '$0.18 vs $21.62 per bug')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 143-bug dataset results from filtering 6,088 Syzbot bugs, but no justification is given for whether 143 is sufficient for the claims made. No power analysis or discussion of statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs per configuration. No standard deviations, variance across seeds, or spread measures are reported in any table or figure."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are included: kGym-oracle with GPT-4-turbo and GPT-4o, kGym-oracle+functionwise, and SimpleAgent without BIC. CrashFixer is discussed as external comparison (Table 1)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "kGym (2024) and CrashFixer (2025) are recent, relevant baselines. The paper also tests with state-of-the-art models (GPT-5 Thinking, Claude Opus 4.1)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 systematically ablates components: function-wise patching, BIC localization, feedback/retries, and function exploration. Section 3.1 explicitly isolates each component's contribution."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: pass rate, bad patch rate, and average cost per bug (Table 1). Compute hours are also provided (Table 3). Manual correctness evaluation adds plausible/helpful/wrong categories (Table 2)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports manual verification of 31 patches, classifying them as plausibly correct (10), helpful (7), or wrong (14). Section 5.1 describes the methodology."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "All 143 bugs are used as the test set for all configurations. There is no explicit dev/test separation, and prompt designs may have been developed using the same bugs that are evaluated."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down reproducer job outputs (pass, trigger, racey, boot fail, other). Table 5 shows build job outputs. Figure 2 shows unique solves per configuration. The dataset targets three bug types (OOB, UAF, NPD)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failure modes are discussed extensively: bad patches, compilation failures (Tables 4-5), non-deterministic reproducers (Section 3.1), and wrong patches that pass crash tests but don't fix the root cause (Table 2, Section 5.1)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Function Exploration Agent achieves only 15.38% pass rate despite being more expensive ($0.12/bug), and Claude Opus 4.1 costs 4.05x more than GPT-5 while performing 11.2% worse. These unfavorable results are reported honestly."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of '43.36% pass rate with GPT-5 Thinking' and 'cost of under $0.20 per bug' are directly supported by Table 1 (43.36% pass rate, $0.18/bug). The claim of 'lightweight, platform-agnostic' is supported by the Docker/QEMU-based architecture."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made via controlled ablation: adding function-wise patching to kGym improves pass rate from 2.8% to 10.49%; adding BIC improves from 17.48% to 21.67%. Each ablation changes a single variable, supporting causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper focuses specifically on Linux kernel KASAN bugs (memory corruption: OOB, UAF, NPD) from Syzbot. The title specifies 'Kernel Program Repair' and claims are consistently scoped to this domain."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed improvements. For example, the paper doesn't consider whether GPT-5's improvement is due to having seen Syzbot fixes in training, or whether the BIC improvement is partly due to the BIC revealing the fix location rather than providing localization insight."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly acknowledges the proxy gap in Section 5.1: of 31 passing patches manually verified, only 32% are plausibly correct and 45% are wrong. They note 'it is insufficient to simply rely on observing the absence of crashes to verify the correctness of patches.'"
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'GPT-4o', 'GPT-4-turbo', 'GPT-5 Thinking', and 'Claude Opus 4.1' without snapshot dates or API version identifiers. No specific version strings like 'gpt-4o-2024-05-13' are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Prompts are described in natural language ('bug-type specific instructions,' 'in-context learning examples for OOB, UAF, and NPD bugs') but the actual prompt text is never shown in the paper or appendix, and no repository link is provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other API parameters are reported for any of the LLM configurations tested."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The scaffolding is described with a workflow diagram (Figure 1), two agent types (Simple Agent and Function Exploration Agent), function-wise patching mechanism, retry logic with error summary feedback, and the RGym test loop. The code-viewing capability of the Function Exploration Agent is described."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The filtering pipeline is documented: 6,088 Syzbot bugs → filter for fix commits, reproducers, crash reports, kernel configs → filter to KASAN bugs → verify reproducibility at parent of fix commit → 143 bugs. Criteria at each stage are stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The conclusion is brief (Section 4) and does not discuss limitations substantively."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper. There is no discussion of threats from non-deterministic reproducers affecting results, potential contamination, or small sample size for manual evaluation."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. It does not discuss exclusion of non-KASAN bugs, non-Linux kernels, or the limited generalizability of results from 143 bugs to kernel APR broadly."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (bug reports, patches generated, evaluation logs) is made available. Only aggregated results in tables are presented."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: bugs sourced from Syzbot, filtered by availability of fix commits, reproducers, crash reports, and kernel configs, restricted to KASAN bugs, and verified for reproducibility. Source (Syzbot) and criteria are stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is Syzbot, a public automated bug reporting system."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While filtering criteria are described, intermediate counts at each stage are missing. The paper jumps from '6,088 Syzbot bugs' to '143 reproducible KASAN bugs' without stating how many bugs remained after each filter step."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper. No grants, sponsors, or funding agencies are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: University of California Riverside and Columbia University. The paper evaluates commercial LLMs (GPT, Claude) but the authors have no disclosed affiliation with those companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence of funders cannot be verified. The paper does not include any statement about funding or competing interests."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4o, GPT-4-turbo, GPT-5 Thinking, or Claude Opus 4.1. This is critical since Syzbot bugs and their fix commits are publicly available."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the LLMs may have seen Syzbot bug reports or their fix commits during training. Syzbot data is publicly indexed and could easily appear in training corpora."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Syzbot bugs and fix commits are publicly available online and predate the models' training. The paper does not address whether the models may have memorized the fixes, which would inflate pass rates."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. This is a benchmark evaluation of LLM-based automated program repair."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on automated kernel bug fixes from Syzbot."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports average cost per bug for every configuration, ranging from $0.05 (kGym+GPT-4o) to $0.73 (Claude Opus 4.1). CrashFixer's cost of $21.62/bug is cited for comparison."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports clock hours for each configuration. Hardware is specified (two 56-core machines, 160GB RAM, 1TB SSD). Section 5.2 describes the compute setup in detail."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs per configuration. No random seed sensitivity analysis or multi-seed results are reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of LLM generation runs per configuration is not explicitly stated. For feedback experiments, retry counts are clear (up to 3), but for non-feedback experiments the paper does not state whether each bug was attempted once or multiple times."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. It is unclear how prompt designs, retry counts, or other configuration choices were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All configurations are reported in Table 1 with their full results, not just the best one. The ablation is systematic and transparent."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement kGym's baseline and report lower numbers than originally published (1.4% vs 5.38%), but do not discuss bias from evaluating their own system against their own baseline reimplementation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 reports both pass rate and cost per bug for each configuration, and Section 3.2 explicitly discusses cost-effectiveness trade-offs. Table 3 provides compute hours."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 explicitly questions whether crash absence equals correctness, showing only 32% of passing patches are plausibly correct. The paper acknowledges 'it is insufficient to simply rely on observing the absence of crashes to verify the correctness of patches.'"
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "When comparing GPT-4o, Claude Opus 4.1, and GPT-5 Thinking, the same SimpleAgent scaffold is used, isolating the model variable. Different scaffolds (SimpleAgent vs Function Exploration) are compared separately."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Syzbot bugs and their fix commits have been publicly available for years. The paper does not discuss whether models trained after these bugs were published could have memorized the fixes."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The BIC (bug-inducing commit) is provided as input, which includes the actual code change that introduced the bug. While the paper argues BICs are obtainable via SymBisect, there is no discussion of whether providing the BIC leaks too much information about the fix."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 143 bugs are independent or whether they share structural similarities (e.g., same subsystems, similar bug patterns) that could bias results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, or decontamination applied."
       }
     }
   },
   "claims": [
     {
       "claim": "SimpleAgent with feedback achieves 37.76% pass rate with GPT-4o at $0.17 per bug",
       "evidence": "Table 1 shows SimpleAgent+Feedback with GPT-4o: 37.76% pass rate, 4.89% bad patch rate, $0.17 avg cost/bug. Section 3.1 details the retry breakdown: 34 bugs first attempt, 8 second, 12 third.",
       "supported": "strong"
     },
     {
       "claim": "GPT-5 Thinking achieves 43.36% pass rate at $0.18 per bug, outperforming Claude Opus 4.1 (32.16% at $0.73)",
       "evidence": "Table 1 directly reports these numbers. Section 3.3 discusses the comparison: GPT-5 costs 4.05x less while performing 11.2% better.",
       "supported": "strong"
     },
     {
       "claim": "Function-wise patching reduces bad patches by 76%",
       "evidence": "Table 1: kGym-oracle bad patch rate drops from 51.88% to 12.14% when function-wise patching is added. Pass rate increases from 2.8% to 10.49%.",
       "supported": "strong"
     },
     {
       "claim": "BIC-based localization improves pass rate by 4.19% over call stack alone",
       "evidence": "Table 1: SimpleAgent-nobic achieves 17.48% vs SimpleAgent with BIC at 21.67%, a 4.19 percentage point improvement (Section 3.1).",
       "supported": "strong"
     },
     {
       "claim": "Combined pass rate of all configurations reaches 68.53%",
       "evidence": "Section 3.3 states this as the union of solved bugs across all configurations at an average cost of $1.33 per bug. Figure 2 shows unique solves per configuration.",
       "supported": "moderate"
     },
     {
       "claim": "kGym's originally reported 5.38% pass rate drops to 1.4% with stricter multi-VM testing",
       "evidence": "Section 3.1 explains the discrepancy: kGym uses a single VM for 10 minutes, but roughly one-third of bugs have non-deterministic reproducers (Table 4). Using 26 VMs reveals this.",
       "supported": "moderate"
     },
     {
       "claim": "Only 32% of passing patches are plausibly correct, with 45% being wrong",
       "evidence": "Table 2 reports manual verification of 31 patches: 10 plausibly correct (32.23%), 7 helpful (22.58%), 14 wrong (45.16%). Section 5.1 describes the verification methodology.",
       "supported": "moderate"
     },
     {
       "claim": "Simple APR designs can achieve comparable results to CrashFixer at 120x lower cost",
       "evidence": "Section 3.3 compares: SimpleAgent GPT-5 at $0.18/bug vs CrashFixer at $21.62/bug (120.11x more expensive). CrashFixer achieves 65.6% vs 43.36%, only 22.24% higher. However, datasets differ and CrashFixer uses oracle localization.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "Syzbot bugs and their fix commits are publicly available and indexed online. LLMs like GPT-5 and Claude Opus 4.1 may have seen these exact bugs and fixes during training. The paper does not address this at all, which could significantly inflate pass rates — especially for GPT-5 Thinking's strong 43.36% result."
     },
     {
       "flag": "No code or data release",
       "detail": "Neither RGym, the APR tool, the curated 143-bug dataset, nor the generated patches are released. This completely prevents independent verification or reproduction of results."
     },
     {
       "flag": "No statistical significance testing",
       "detail": "All comparisons between configurations are based on raw pass rate differences (e.g., '4.19% improvement') without any significance tests. With only 143 bugs, many of these differences may not be statistically significant."
     },
     {
       "flag": "Small manual correctness sample",
       "detail": "Only 31 of the passing patches were manually verified for correctness (Table 2). This is a small sample for drawing conclusions about the overall correctness distribution, especially when extrapolating to other configurations or models."
     },
     {
       "flag": "Indirect comparison with CrashFixer",
       "detail": "The paper compares against CrashFixer's reported numbers on kBenchSyz, described only as 'similar enough' to their dataset. The datasets differ, CrashFixer uses oracle localization, and CrashFixer is closed source — making the comparison unreliable for strong claims about relative performance."
     },
     {
       "flag": "Single-run results without variance",
       "detail": "All configurations appear to be evaluated once. Given that LLM outputs are stochastic and reproducers are non-deterministic (one-third of bugs per Table 4), results could vary significantly across runs."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Major LLM coding benchmark that the paper positions kernel APR as complementing, since SWE-bench focuses on userspace applications."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2025,
       "relevance": "Autonomous LLM-based program repair agent, directly relevant as a concurrent approach to automated bug fixing."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik R. Narasimhan", "Ofir Press"],
       "year": 2024,
       "relevance": "Foundational agent-computer interface framework for LLM-based software engineering, cited as part of the APR landscape."
     },
     {
       "title": "Kgym: A platform and dataset to benchmark large language models on linux kernel crash resolution",
       "authors": ["Alex Mathai", "Chenxi Huang", "Petros Maniatis", "Aleksandr Nogikh", "Franjo Ivancic", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2024,
       "arxiv_id": "2407.02680",
       "relevance": "Direct predecessor: the platform and dataset that RGym improves upon for kernel APR evaluation."
     },
     {
       "title": "CrashFixer: A crash resolution agent for the linux kernel",
       "authors": ["Alex Mathai", "Chenxi Huang", "Suwei Ma", "Jihwan Kim", "Hailie Mitchell", "Aleksandr Nogikh", "Petros Maniatis", "Franjo Ivančić", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2025,
       "relevance": "Primary comparison point: complex debug-tree-based kernel APR achieving 65.6% pass rate at $21.62/bug using oracle localization."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2025,
       "relevance": "Open platform for AI software engineering agents, part of the broader LLM-based development tool ecosystem."
     },
     {
       "title": "InferFix: End-to-end program repair with LLMs",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano"],
       "year": 2023,
       "relevance": "End-to-end LLM-based program repair system, part of the APR literature this work builds upon."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Demonstrates cost-effective conversational APR with ChatGPT, directly relevant to the cost-efficiency claims in this paper."
     },
     {
       "title": "How far can we go with practical function-level program repair?",
       "authors": ["Jiahong Xiang", "Xiaoyang Xu", "Fanchu Kong"],
       "year": 2024,
       "relevance": "Explores function-level program repair, directly relevant to the function-wise patching technique used in RGym."
     },
     {
       "title": "Aligning the Objective of LLM-Based Program Repair",
       "authors": ["Junjielong Xu", "Ying Fu", "Shin Hwei Tan", "Pinjia He"],
       "year": 2025,
       "relevance": "Addresses alignment of LLM objectives for program repair, relevant to the broader APR methodology."
     },
     {
       "title": "A case study of LLM for automated vulnerability repair: Assessing impact of reasoning and patch validation feedback",
       "authors": ["Ummay Kulsum", "Haotian Zhu", "Bowen Xu", "Marcelo d'Amorim"],
       "year": 2024,
       "relevance": "Studies feedback-based LLM vulnerability repair, directly relevant to the retry/feedback mechanism in this paper."
     },
     {
       "title": "Boosting automated program repair with bug-inducing commits",
       "authors": ["Ming Wen", "Yepang Liu", "Shing-Chi Cheung"],
       "year": 2020,
       "relevance": "Foundational work on using bug-inducing commits for APR localization, the key localization strategy adopted by RGym."
     },
     {
       "title": "SymBisect: Accurate bisection for Fuzzer-Exposed vulnerabilities",
       "authors": ["Zheng Zhang", "Yu Hao", "Weiteng Chen"],
       "year": 2024,
       "relevance": "Automated bug-inducing commit identification tool achieving 75% accuracy on Syzbot bugs, key enabler for the realistic localization strategy."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "RGym is designed for local commodity hardware, making kernel APR evaluation more accessible, though the tool itself is not released."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Challenges the assumption that kernel APR requires complex pipelines like CrashFixer, showing a simple approach achieves comparable results at 120x lower cost."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper focuses on fixing kernel bugs, not attacking systems."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Implicitly critiques kGym's GCP dependency and CrashFixer's complexity/cost, but the tone is measured and academic."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, data, or demo is released; the tool cannot be tried by readers."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors are from UC Riverside and Columbia; tests GPT-5 and Claude Opus which are recognizable but the paper/lab is not from a major AI company."
     }
   }
 }