ai-research-survey

scan.json (32172B)

---

 {
   "paper": {
     "title": "How Far Can We Go with Practical Function-Level Program Repair?",
     "authors": [
       "Jiahong Xiang",
       "Xiaoyang Xu",
       "Fanchu Kong",
       "Mingyuan Wu",
       "Zizheng Zhan",
       "Haotian Zhang",
       "Yuqun Zhang"
     ],
     "year": 2024,
     "venue": "arXiv.org",
     "arxiv_id": "2404.12833",
     "doi": "10.48550/arXiv.2404.12833"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLMs with zero-shot learning are already powerful function-level APR techniques, while few-shot learning produces disparate and sometimes negative results (from +10% to -49.7% across models). Auxiliary repair-relevant information (trigger tests, error messages, comments) significantly improves repair performance, nearly matching costly statement-level fault location. SRepair, a dual-LLM framework using Chain of Thought for repair suggestions, correctly fixes 300 single-function and 32 multi-function bugs in Defects4J, outperforming prior APR techniques by at least 85%.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'The data and code are available at GitHub [1]' referencing https://github.com/GhabiX/SRepair. A concrete URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available Defects4J (v1.2 and v2.0) and QuixBugs datasets. The GitHub repository also provides supplementary data."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (128-core AMD EPYC 7H12, 512 GiB RAM, 8x NVIDIA A100 80GB, Ubuntu 20.04.6 LTS) but no software environment details (requirements.txt, dependency versions, Dockerfile) are provided in the paper. The paper defers to GitHub for 'LLM configuration details' but does not describe a reproducible software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper references GitHub for code and data but does not include step-by-step reproduction instructions. The experimental pipeline (patch generation, validation) is described at a high level but not as executable steps."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (number of plausible/correct fixes). No confidence intervals, error bars, or uncertainty measures are provided in any tables or figures."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'surpassing all previous APR techniques by at least 85%') based solely on comparing raw counts without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '85% more than ChatRepair' (from 162 to 300), '1.59× more than Repilot,' and specific percentage changes for each setup comparison (e.g., '26.7% improvement' for trigger tests)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the sample size of 200 patches per bug (beyond 'following prior works') or for the choice of 522 bugs. No power analysis or sample size reasoning is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. Results are single-run numbers. The 200 samples per bug use stochastic sampling (temperature=0.8) but no variance across runs is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four recent SOTA LLM-based APR techniques are used as baselines: AlphaRepair, Repilot, FitRepair, and ChatRepair. Additionally, GPT-3.5-Turbo and Magicoder with PI(ALL) serve as ablation baselines."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are recent: AlphaRepair (2022), Repilot (2023), FitRepair (2023), ChatRepair (2023). These represent the state of the art in LLM-based APR at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper includes ablation variants: SRepair2M (dual-LLM without CoT), SRepair2M+FL (with fault location), and separate evaluations of GPT-3.5-Turbo and Magicoder components. The study section also systematically varies K-shot settings and auxiliary information types."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two primary metrics are used: plausible fixes (patches passing all tests) and correct fixes (manually verified semantic correctness). Patch distribution (plausible/test-failure/uncompilable ratios) is also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Three authors cross-validated the plausible patches of SRepair500 for semantic equivalency to determine correct patches, as stated in Section 5.2.3."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The Defects4J test suite (trigger tests and relevant tests) serves as the evaluation set. Since models are used in zero-shot mode without any tuning on these bugs, the test data is not used for any selection decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 provides per-project breakdowns for all 17 Defects4J projects. Results are shown separately for D4J 1.2 and D4J 2.0. Tables 3-5 provide per-model breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Multiple failure cases are discussed: Closure-66 (bug report misleads LLM, Figure 6b), Closure-112 (fault location information misleads LLM, Figure 7), and CodeLlama generating Python code for Java bugs (Section 4.2)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: few-shot learning causes up to 49.7% decline (Magicoder in K2(CE,PE)), fault location can mislead and prevent 14 previously fixable bugs from being fixed, and bug reports cause 19 bugs to become unfixable."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 300 correct single-function fixes confirmed in Table 7, 85% improvement over ChatRepair (300 vs 162), 32 multi-function fixes shown in Figure 11, and the study findings about zero-shot and auxiliary information are demonstrated in Tables 3-5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims about few-shot learning and auxiliary information effects are supported by controlled experiments varying one factor at a time across the same models and bugs. The ablation variants (SRepair2M, SRepair2M+FL, SRepair200) also use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'How Far Can We Go with Practical Function-Level Program Repair?' and findings (e.g., 'LLMs with zero-shot learning are already powerful function-level APR techniques') are stated broadly, but results are primarily from Java bugs in Defects4J. QuixBugs adds Python but is only 40 trivial bugs. The generalization to other languages and real-world bug distributions is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section addresses data leakage and manual validation but does not discuss alternative explanations for the core findings. For example, whether SRepair's improvement comes primarily from additional context tokens rather than the CoT mechanism, or whether the dual-LLM architecture benefits from two independent sampling chances, is not discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper distinguishes between plausible patches (pass all tests, the proxy) and correct patches (manually verified semantic equivalence, the desired outcome). Section 5.2.3 explicitly notes the manual inspection process, acknowledging the gap between test-passing and correctness."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are stated: 'code-davinci-edit-001' for Codex-edit, 'gpt-3.5-turbo-1106' for GPT-3.5-Turbo, CodeLlama-Instruct 7B/13B/34B, and 'MagicoderS-CL' for Magicoder. These are specific enough to identify the exact models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Figure 3 shows a prompt template with placeholders ('{Buggy code and fixed code pair examples}', '{Auxiliary repair-relevant Information}'). Figure 9 shows the CoT prompt structure. However, the full prompt text with actual fill values is not reproduced. The paper defers configuration details to the GitHub page."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are stated in Section 3.3: 'nucleus sampling with top p = 0.9, temperature = 0.8 and 200 samples per bug.' SRepair generates 5 patched functions per repair suggestion. A 5-hour time limit per bug is also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The SRepair dual-LLM framework is described in detail in Section 5.1 with Figure 8: GPT-3.5-Turbo as the repair suggestion model using CoT, Magicoder as the patch generation model, with the workflow of analyzing buggy code → identifying root cause → generating repair suggestions → generating patched functions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.2 describes benchmark construction: 522 single-function bugs extracted from Defects4J 1.2 and 2.0, including 276 single-hunk and 158 single-line bugs. Section 3.3.3 details collection of bug reports from official issue links, extraction of trigger tests, error messages, and comments."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated section with three subsections: threats to internal validity, external validity, and construct validity."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: three authors cross-validated patches (internal), only 7.4‰ of patches match developer patches for data leakage (internal), dual-LLM mitigates trigger test manipulation (internal), Defects4J may not generalize (external), and evaluation extended to QuixBugs and multi-function bugs to mitigate (external)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The external validity discussion notes that 'evaluation datasets used may not well generalize' but does not explicitly state what the results do NOT show. No specific exclusions are listed (e.g., no mention that results may not apply to languages other than Java, industrial codebases, or bugs beyond those in curated benchmarks)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Defects4J is publicly available. The paper states 'The data and code are available at GitHub' with a URL. The underlying bug data, test suites, and project code are all accessible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1.2 describes collecting 522 single-function bugs from Defects4J 1.2 and 2.0. Section 3.3.3 describes how bug reports were collected from official issue links, trigger tests extracted by building projects, and error messages captured from JUnit. Table 1 provides full statistics."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is Defects4J, a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: bugs selected from Defects4J → auxiliary info collected (bug reports, trigger tests, error messages, comments) → prompts constructed → patches generated via nucleus sampling → patches compiled and tested → plausible patches manually inspected for correctness. Counts are provided at key stages (522 bugs, 200 samples per bug, >10M total patches)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. The absence of any funding disclosure leaves it unclear whether the work was funded."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Southern University of Science and Technology (academic) and Kwai Inc. (industry). Both affiliations are displayed prominently with the author names."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since funding is not disclosed, independence of the funder cannot be assessed. Two authors are affiliated with Kwai Inc. but the paper does not evaluate Kwai products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the six models used (Codex-edit, GPT-3.5-Turbo, CodeLlama 7B/13B/34B, Magicoder)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses 'potential for data leakage if the developer patches were included in the original training data' and reports that only 7.4‰ of plausible patches in the study and 1.5‰ from SRepair500 are identical to developer patches."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses contamination by checking the identity between generated plausible patches and developer patches (7.4‰ and 1.5‰ overlap rates). However, it does not discuss the temporal availability of Defects4J (published 2014) relative to model training, making the analysis incomplete but present."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLM-based program repair."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports 'repairing 300 single-function bugs with SRepair costs only $8.6, averaging $0.029 per correct fix.' The study section also reports '>10 million patches generated and validated, consuming more than 8,000 GPU and 100,000 CPU hours.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total compute is stated: 'more than 8,000 GPU and 100,000 CPU hours,' hardware specified (128-core AMD EPYC 7H12, 8x NVIDIA A100 80GB), and the 5-hour per-bug time limit for SRepair."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are not reported across multiple random seeds. The 200 samples per bug use stochastic sampling (temperature=0.8) but no seed sensitivity analysis is provided and no variance across independent runs is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of samples is explicitly stated: '200 samples per bug' for the study and SRepair200, and SRepair500 generates 500 samples. SRepair generates '5 patched functions' per repair suggestion."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (top_p=0.9, temperature=0.8) are stated as 'following prior works' but no search budget or justification for these specific values is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The selection of GPT-3.5-Turbo for repair suggestions and Magicoder for patch generation is justified based on empirical results from the study section (Section 4.2). All K-shot and auxiliary information configurations are reported, not just the best."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare SRepair against published results of other techniques (AlphaRepair, Repilot, FitRepair, ChatRepair) without acknowledging self-evaluation bias or the possibility that their implementation/evaluation environment might favor their own approach."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While SRepair200 and SRepair500 results are shown separately, there is no systematic analysis of performance as a function of compute budget. The baselines (AlphaRepair, Repilot, etc.) may use different compute budgets but this is not discussed as a confound."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether Defects4J bugs are representative of real-world bug distributions, or whether the plausible-patch metric (passing all tests) adequately captures repair quality beyond the correct/plausible distinction."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Baselines use fundamentally different architectures (AlphaRepair: cloze-style, Repilot: LSP-guided, ChatRepair: conversational, SRepair: dual-LLM CoT). The comparison attributes performance differences to the approach without isolating scaffold from model effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses 'potential for data leakage if the developer patches were included in the original training data' and checks patch identity rates (7.4‰ and 1.5‰). While the temporal relationship between Defects4J (2014) and model training is not explicitly stated, the core concern is addressed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses that 'the LLMs might have recognized the trigger tests and manipulated them to pass all tests, creating seemingly plausible patches.' The dual-LLM mechanism is described as mitigating this: the repair suggestion model only sees auxiliary info, keeping the patch generation model isolated from trigger test data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the source code of Defects4J projects (Apache Commons, Closure Compiler, etc.) appears in the training data of the models, which would create non-independence between training and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A concrete detection method is applied: exact match analysis between generated plausible patches and developer patches, finding only 7.4‰ identity rate across the study and 1.5‰ for SRepair500."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs with zero-shot learning are already powerful function-level APR techniques, achieving 180 average plausible fixes and outperforming all few-shot setups by at least 10.4%.",
       "evidence": "Table 3 shows K0(Basic) achieves 180 average plausible fixes vs. K1(CE) at 149, K1(PE) at 163, K2(CE,PE) at 149, and K2(PE,PE) at 160. Section 3.4.1.",
       "supported": "strong"
     },
     {
       "claim": "Few-shot learning leads to disparate and even negative impacts on function-level repair, ranging from +10% to -49.7% across models.",
       "evidence": "Table 3: CodeLlama 34B shows +10% (176 vs 160) in K1(CE), while Magicoder shows -49.7% (100 vs 199) in K2(CE,PE). Section 3.4.1.",
       "supported": "strong"
     },
     {
       "claim": "Directly applying trigger tests and error messages significantly enhances repair performance by 26.7% and 26.1% respectively.",
       "evidence": "Table 4 shows PI(TT) achieves 228 vs 180 for K0(Basic) (26.7% increase) and PI(EM) achieves 227 vs 180 (26.1% increase). Section 3.4.2.",
       "supported": "strong"
     },
     {
       "claim": "SRepair correctly fixes 300 single-function bugs, surpassing ChatRepair by 85% and Repilot by 1.59x, without statement-level fault location.",
       "evidence": "Table 7 shows SRepair500 achieves 300 correct fixes vs ChatRepair at 162. Section 5.2.5.",
       "supported": "moderate"
     },
     {
       "claim": "SRepair successfully fixes 32 multi-function bugs, the first time achieved by any APR technique.",
       "evidence": "Figure 11 shows SRepair500 achieves 53 plausible and 32 correct fixes for multi-function bugs. Figure 12 shows the JacksonDatabind-69 multi-function example. Section 5.2.5.",
       "supported": "weak"
     },
     {
       "claim": "Auxiliary repair-relevant information can nearly replace costly statement-level fault location, with only a 7.1% gap.",
       "evidence": "Table 5 shows PI(ALL) achieves 254 plausible fixes vs PI(ALL)+FL at 272, a 7.1% gap. Section 3.4.2.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('surpassing by 85%', 'significantly enhances') are based on comparing raw counts without any statistical significance tests. With stochastic sampling at temperature=0.8, results could vary across runs, but no variance is reported."
     },
     {
       "flag": "No variance across runs",
       "detail": "Despite using stochastic sampling (temperature=0.8, top_p=0.9), all results are reported as single-run point estimates. No standard deviations, confidence intervals, or cross-run variance is provided. The stability of the reported 300 correct fixes is unknown."
     },
     {
       "flag": "'First ever' claim unverifiable",
       "detail": "The claim that SRepair is 'the first time achieved by any APR technique ever' for multi-function bug repair is extremely strong and difficult to comprehensively verify against all prior work."
     },
     {
       "flag": "Defects4J contamination risk understated",
       "detail": "Defects4J has been public since 2014 and is extensively discussed in SE literature. The contamination analysis only checks exact patch identity (7.4‰) but does not address whether models saw Defects4J project code or bug discussions during training. Training cutoff dates are not stated for any model."
     },
     {
       "flag": "Scaffold confound in baseline comparison",
       "detail": "SRepair uses a dual-LLM CoT framework while baselines use fundamentally different architectures (cloze-style, conversational, LSP-guided). The 85% improvement claim does not isolate whether gains come from the methodology, the additional information, or the increased compute (500 samples with dual models)."
     }
   ],
   "cited_papers": [
     {
       "title": "Keep the Conversation Going: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2304.00385",
       "relevance": "State-of-the-art conversational LLM-based APR technique (ChatRepair) and primary comparison baseline."
     },
     {
       "title": "Less training, more repairing please: revisiting automated program repair via zero-shot learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "AlphaRepair: zero-shot APR using CodeBERT, key baseline for cloze-style repair."
     },
     {
       "title": "Copiloting the copilots: Fusing large language models with completion engines for automated program repair",
       "authors": ["Yuxiang Wei", "Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Repilot: LLM+completion engine for compilable patches, key baseline for function-level APR."
     },
     {
       "title": "Revisiting the Plastic Surgery Hypothesis via Large Language Models",
       "authors": ["Chunqiu Steven Xia", "Yifeng Ding", "Lingming Zhang"],
       "year": 2023,
       "arxiv_id": "2303.10494",
       "relevance": "FitRepair: LLM-based APR combining fine-tuning with prompting strategies."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Study of LLM-based APR with few-shot learning, establishing the function-level APR investigation that this paper builds on."
     },
     {
       "title": "Automated repair of programs from large language models",
       "authors": ["Zhiyu Fan", "Xiang Gao", "Martin Mirchev", "Abhik Roychoudhury", "Shin Hwei Tan"],
       "year": 2023,
       "relevance": "Study of repairing LLM-generated program errors, exploring APR in LLM code generation context."
     },
     {
       "title": "Inferfix: End-to-end program repair with llms",
       "authors": ["Matthew Jin", "Syed Shahriar", "Michele Tufano", "Xin Shi", "Shuai Lu", "Neel Sundaresan", "Alexey Svyatkovskiy"],
       "year": 2023,
       "arxiv_id": "2303.07263",
       "relevance": "End-to-end LLM-based APR technique using few-shot learning."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "doi": "10.48550/arxiv.2107.03374",
       "relevance": "Codex evaluation paper establishing LLM code generation capabilities and nucleus sampling methodology."
     },
     {
       "title": "An empirical study on fine-tuning large language models of code for automated program repair",
       "authors": ["Kai Huang", "Xiangxin Meng", "Jian Zhang", "Yang Liu"],
       "year": 2023,
       "relevance": "Empirical study of fine-tuning LLMs for APR, directly relevant to understanding LLM repair capabilities."
     },
     {
       "title": "Rap-gen: Retrieval-augmented patch generation with codet5 for automatic program repair",
       "authors": ["Weishi Wang", "Yue Wang", "Shafiq Joty", "Steven CH Hoi"],
       "year": 2023,
       "relevance": "Retrieval-augmented approach to LLM-based APR, combining retrieval with generation for patch generation."
     },
     {
       "title": "GAMMA: Revisiting template-based automated program repair via mask prediction",
       "authors": ["Quanjun Zhang", "Chunrong Fang", "Tongke Zhang"],
       "year": 2023,
       "relevance": "Template-based APR using mask prediction, representing alternative approach to LLM-based repair."
     },
     {
       "title": "Selfapr: Self-supervised program repair with test execution diagnostics",
       "authors": ["He Ye", "Matias Martinez", "Xiapu Luo", "Tao Zhang", "Martin Monperrus"],
       "year": 2022,
       "relevance": "Self-supervised APR using test execution diagnostics, relevant to using test information for repair."
     },
     {
       "title": "Can OpenAI's codex fix bugs? an evaluation on QuixBugs",
       "authors": ["Julian Aron Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "relevance": "Early evaluation of Codex for APR on QuixBugs, establishing LLM repair evaluation methodology."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "SRepair is a usable APR technique with released code, but requires API access to GPT-3.5-Turbo and local GPU for Magicoder."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that zero-shot outperforms few-shot for function-level APR is mildly surprising but aligns with emerging understanding of LLM in-context learning."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns raised; this is a constructive software engineering tool paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; straightforward empirical evaluation and technique proposal."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code released on GitHub with data, though reproducing requires API access and GPU resources."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses GPT-3.5-Turbo (recognizable) but authors are from Southern University of Science and Technology and Kwai Inc., not top-tier AI labs."
     }
   }
 }