ai-research-survey

scan.json (33234B)

---

 {
   "paper": {
     "title": "Explainable Automated Debugging via Large Language Model-driven Scientific Debugging",
     "authors": [
       "Sungmin Kang",
       "Bei Chen",
       "Shin Yoo",
       "Jian-Guang Lou"
     ],
     "year": 2023,
     "venue": "Empirical Software Engineering",
     "arxiv_id": "2304.02195",
     "doi": "10.1007/s10664-024-10594-x"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "qualitative"],
   "key_findings": "AutoSD, which uses LLMs to emulate Scientific Debugging with iterative hypothesis-observe-conclude loops and real debugger interaction, achieves competitive automated program repair performance (187 correct on ARHE, 113 on Defects4J v2.0) while generating explainable reasoning traces. A human study with 20 participants showed explanations improved patch review accuracy on 5 of 6 real-world bugs without increasing review time. The <DONE> token reliably indicates higher confidence in generated patches, and debugger ablation reduced plausible patch rate from 73% to 63%. Professional developers were notably less satisfied with explanations than students, suggesting need for tighter integration with development workflows.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.1 states 'we plan to make our implementation and repair results publicly available for scrutiny' — this is a promise of future release, not an actual release. No repository URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The ARHE dataset (200 mutated bugs from HumanEval) is newly constructed but not released — only promised for future availability. Defects4J and BugsInPy are public benchmarks, but the human study data and ARHE mutations are not available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment specification is provided. The paper mentions using jdb, pdb, ChatGPT, Codex, and CodeGen but does not specify library versions or environment setup."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The technique pipeline is described at a conceptual level (Section 3) and the full prompt is in the appendix, but there are no runnable scripts or setup instructions."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The template-based baseline reports mean ± std (85.77 ± 4.20 in Table 1), but the main LLM-based results (LLM-Base and AutoSD) are point estimates only with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "For the human study time comparison, the paper states 'There is no case where the difference is statistically significant,' implying significance tests were run on the time data. However, no significance tests are reported for the benchmark APR performance comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are provided throughout: absolute numbers (187 vs 177 correct, Table 1), percentage point differences ('12.4%p more likely to be plausible,' Section 5.2), and per-bug accuracy breakdowns in the human study (Figure 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The human study uses N=20 with no power analysis or justification for why 20 participants were chosen. The benchmark sizes (200 ARHE bugs, Defects4J) are standard but not justified either."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only the template-based baseline reports variance (100 reruns with std). The LLM-based methods (AutoSD and LLM-Base) report single-run results with 10 patches per bug but no variance across runs or seeds."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple baselines are compared: LLM-Base (direct LLM fix), template-based APR baseline (reverse mutators), Recoder (DL-based APR), and finetuned InCoder from Jiang et al. (Tables 1-2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Recoder and InCoder results from Jiang et al. (2023) and Codex results from Xia et al. (2022), which are contemporary state-of-the-art APR techniques at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ2 provides a debugger ablation study comparing AutoSD with real debugger execution vs. 'hallucinated' observations, showing that real execution improves plausible patch rate from 63% to 73% and reverses the <DONE> confidence signal."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are reported: plausible patches, correct patches (Tables 1-2), developer accuracy, developer time, and subjective helpfulness ratings (Figure 5) in the human study."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "A human study with 20 participants (including 6 professional developers) evaluates patch review accuracy and time with and without AutoSD explanations (Section 4.2.2, RQ4-RQ5)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The ARHE dataset was specifically constructed to avoid data contamination by mutating HumanEval, which was designed to avoid overlap with training data. Defects4J is a standard held-out benchmark with separate test suites."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (ARHE, D4J v1.2, v2.0 in Tables 1-2), per bug in the human study (Figure 5), by <DONE> vs no-<DONE> status (Figure 3), and by participant group (students vs professionals, Figure 6)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "RQ6 (Section 5.6) provides qualitative failure analysis including a disliked example (BIP002) and systematic analysis of 25 failure cases, identifying that 13/25 failures were due to breakpoints not being hit."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Multiple negative results: explanations decreased accuracy for ARHE105 and BIP003 (Section 5.4), professional developers were largely unsatisfied with AutoSD (Figure 6b), and AutoSD takes 5x longer than LLM-Base (Section 6.2)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: 'performs competitively' (Tables 1-2), 'can indicate when confident' (Figure 3), 'accuracy improved for five out of six real-world bugs' (Figure 5), '70% answered they wanted explanations' and '55% satisfied' (Figure 6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim (explanations improve accuracy) is supported by a within-subjects randomized design where participants see 3 bugs with and 3 without explanations in random order. The ablation study uses controlled single-variable manipulation (debugger vs no debugger)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to the tested settings. The paper specifies 'on three program repair benchmarks' and reports specific numbers like 'five out of six real-world bugs studied.' Limitations explicitly note 'our technique can only handle single-method bugs' (Section 6.2)."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The threats to validity section (6.1) discusses potential confounds like incorrect implementations and biased responses, but does not substantively discuss alternative explanations for the results — e.g., whether the performance gain is from extra compute rather than the scientific debugging structure, or whether explanation effects are driven by anchoring bias."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures patch correctness review accuracy as a proxy for real-world APR utility, and explicitly discusses the distinction between student vs professional performance and between lab study and real deployment. The patch review task is directly motivated by actual industry practice at Meta and Bloomberg (Section 2.1)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Codex is specified as 'code-davinci-002' but the default model ChatGPT is described only as 'a sibling model to InstructGPT' without a specific version, snapshot date, or API version. CodeGen is identified only by size (6B) without a specific checkpoint."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full Scientific Debugging description prompt is provided in the appendix (Section 4 of the appendix), including detailed examples for hypotheses, predictions, experiments, observations, and conclusions. The fix generation prompt template is also given in Section 3.3."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The maximum iteration limit s=3 and 10 patches per bug are stated, but critical LLM API parameters (temperature, top-p, max tokens, frequency penalty) are not reported for any of the models used."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The AutoSD pipeline is described in detail in Section 3: prompt construction (3.1), hypothesize-observe-conclude loop (3.2) with DSL commands (REPLACE, ADD, DEL, RUN), debugger interface (jdb/pdb), and fix suggestion (3.3). Figure 1 provides a complete pipeline diagram."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "ARHE construction is documented: 7 mutators applied to HumanEval solutions yielding 200 bugs (Appendix Table 1). Human study design is documented: 12 bugs sampled, divided into 2 groups of 6, randomly assigned to participants (Section 4.2.2)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Discussion' contains both 6.1 'Threats to Validity' and 6.2 'Limitations' with substantive discussion of multiple issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: potential training data overlap mitigated by constructing ARHE (external validity), the 5x time cost (Section 6.2), single-method bug limitation, method-level FL assumption, and the risk of credibility lending to incorrect patches."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicit scope boundaries: 'our technique can only handle single-method bugs as of now' (Section 6.2), 'we evaluated in the setting where method-level FL was done' (Section 6.2), and the maximum iteration limit s=3 constraining debugging depth."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data (generated patches, debugging traces, human study responses) are not available. The paper only promises future release: 'we plan to make our implementation and repair results publicly available for scrutiny' (Section 6.1)."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "ARHE construction via 7 mutators on HumanEval is detailed (Section 4.2.1, Appendix Section 2). Human study procedure is described: recruitment channels, task structure, practice problem, 30-40 minute sessions, post-questionnaire, and 5-minute interviews (Section 4.2.2)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.2: 'we advertised the task to both undergraduate and graduate students with at least 1 year of Python experience, as well as professional developers at a company that specializes in software testing techniques.' Resulting sample: 8 undergrad, 6 graduate, 6 professional developers."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The ARHE construction pipeline is documented with mutator breakdown (Appendix Table 1, 200 bugs). The human study pipeline is documented: 12 bugs sampled → divided into 2 groups of 6 → randomly assigned participants → 6 problems per participant with 3 having explanations."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No formal funding or acknowledgments section is present. The footnote mentions 'This work was done as part of an internship at Microsoft Research Asia' but there is no explicit funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Sungmin Kang and Shin Yoo at KAIST, Bei Chen and Jian-Guang Lou at Microsoft Research Asia. The internship arrangement is also disclosed."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Two of four authors are at Microsoft Research Asia, and the work was done during an MSRA internship. Microsoft is a major investor in OpenAI, whose ChatGPT/Codex models are the primary tools evaluated. Microsoft has a financial interest in demonstrating LLM utility for developer tools."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Microsoft's investment in OpenAI and potential commercial interest in LLM-based developer tools are not disclosed."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (ChatGPT, Codex, CodeGen). This is relevant because the Defects4J benchmarks could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 (External Validity) explicitly discusses the concern: 'A particular concern when using large language models is that their training data may include segments of the evaluation data.' The ARHE dataset was specifically constructed to mitigate this using HumanEval, which was designed to avoid contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper constructs ARHE from HumanEval specifically because 'HumanEval was explicitly made by Chen et al. to avoid data contamination when evaluating their LLM' (Section 4.2.1). However, contamination for Defects4J is only acknowledged as a threat, not actively tested."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the human study. No link to OSF, AsPredicted, or other pre-registration platform."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.2 states: 'Our human study received IRB review exemption (IRB-23-054).'"
       },
       "demographics_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.2 reports: 'eight undergraduate and six graduate students, as well as six professional developers whose career span from 3 to 10 years.' Python experience requirement (≥1 year) is also stated."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": true,
         "justification": "Inclusion criteria stated: 'undergraduate and graduate students with at least 1 year of Python experience, as well as professional developers at a company that specializes in software testing techniques' (Section 4.2.2)."
       },
       "randomization_described": {
         "applies": true,
         "answer": true,
         "justification": "Randomization is described: bugs divided into 2 groups, participants 'randomly assigned' to a group, explanations provided for 'a randomly selected three of the six cases,' and problems solved 'in a randomized order' (Section 4.2.2)."
       },
       "blinding_described": {
         "applies": true,
         "answer": false,
         "justification": "No blinding is described. Participants could see whether an explanation was available or not (the explanation panel was visibly present or absent). No mention of blinding for evaluators assessing patch correctness."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No explicit attrition reporting. Twenty participants were recruited and results appear to include all 20, but the paper does not explicitly state that all participants completed the study or report dropout."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only relative time is mentioned: 'AutoSD could take about five times longer to generate a patch when compared to LLM-Base' (Section 6.2). No absolute API costs, token counts, or per-example costs are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report total API spend, number of API calls, or total tokens consumed across the experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The template-based baseline is run 100 times with std reported, but the LLM-based methods (AutoSD and LLM-Base) do not report results across multiple seeds or runs. LLM stochasticity is not addressed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2.1 states: 'for each dataset we provide AutoSD with the buggy method and generate 10 patches.' The template-based baseline uses '100 reruns' (Table 1)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The paper states ChatGPT was chosen because 'we empirically found the best performance' (Section 5.3) but does not describe the selection process or configurations tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "ChatGPT is selected as the default model because it showed 'best performance' (Section 5.3), but the selection criteria, validation set, and process for this determination are not described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The human study compares accuracy across 12 bugs with and without explanations, and the paper reports significance tests for time differences, but no correction for multiple comparisons is mentioned."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement both AutoSD and LLM-Base themselves. They compare against external baselines (Recoder, InCoder from Jiang et al.) but do not acknowledge author-evaluation bias in their own implementation comparisons."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "AutoSD takes '5 times longer' than LLM-Base (Section 6.2) due to iterative LLM and debugger calls, but performance is not compared at matched compute budgets. The performance comparison ignores this significant compute difference."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether plausible/correct patch counts on ARHE and Defects4J adequately measure the claimed 'competitive repair performance.' The ARHE dataset's construct validity (mutation-based bugs vs real-world bugs) is not analyzed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "In RQ3 (Figure 4), the same scaffolding (AutoSD) is used across different LLMs (CodeGen, Codex, ChatGPT) and compared against the same LLM-Base scaffold, isolating the model variable from the scaffold variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While ARHE is constructed from HumanEval to mitigate contamination, temporal leakage is not explicitly discussed. Defects4J bugs (2014-era) predate all model training cutoffs but this temporal relationship is not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The buggy function, failing test, and error message are all provided to the model, but whether this mirrors realistic information availability is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark bugs share structural similarities that could inflate results, or whether ARHE bugs drawn from the same HumanEval problems introduce non-independence."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method is applied. ARHE construction mitigates contamination by design, but no canary strings, membership inference, or n-gram overlap tests are used to verify the absence of leakage."
       }
     }
   },
   "claims": [
     {
       "claim": "AutoSD achieves competitive program repair performance compared to non-explainable APR techniques.",
       "evidence": "Table 1: 187 correct on ARHE vs 177 for LLM-Base. Table 2: 113 correct on D4J v2.0 vs 110 for LLM-Base, and outperforms Recoder (11) and InCoder (28). Also outperforms Codex with 200 candidates under comparable FL settings (Section 5.1).",
       "supported": "moderate"
     },
     {
       "claim": "The <DONE> token prediction reliably indicates higher confidence and better repair performance.",
       "evidence": "Figure 3: Higher plausible and correct patch proportions when <DONE> is predicted on both ARHE and Defects4J. 89% of <DONE> plausible patches were correct vs 82% without <DONE>. The signal reverses when debugger is ablated (Section 5.2).",
       "supported": "moderate"
     },
     {
       "claim": "Using real debugger execution rather than hallucinated observations improves reliability.",
       "evidence": "Section 5.2: Individual runs were 73% plausible with real execution vs 63% with hallucinated observations. <DONE> token becomes a negative signal (11%p less likely plausible) under hallucination, vs a positive signal (12.4%p more likely) with real execution.",
       "supported": "strong"
     },
     {
       "claim": "AutoSD performance rapidly improves with stronger language models.",
       "evidence": "Figure 4: Performance increases from near-zero (CodeGen-6B) to competitive levels (ChatGPT) as model capability improves. CodeGen-6B fails in zero-shot but fixes 44 bugs few-shot (Section 5.3).",
       "supported": "moderate"
     },
     {
       "claim": "Explanations from AutoSD improve developer accuracy on real-world bug patch review.",
       "evidence": "Figure 5: Accuracy improved in 5 of 6 BugsInPy (real-world) bugs when explanations were available. However, accuracy decreased for ARHE105 and BIP003. Time to review was roughly constant (Section 5.4).",
       "supported": "moderate"
     },
     {
       "claim": "70% of participants want explanations when using APR tools, and 55% find Scientific Debugging formulation satisfactory.",
       "evidence": "Figure 6: Post-questionnaire results show 70% agreed explanations are important for APR. 55% were satisfied with the Scientific Debugging details. Notable split between students (more satisfied) and professional developers (more critical) in Section 5.5.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Conflict of interest not disclosed",
       "detail": "Two of four authors are at Microsoft Research Asia, and the work was done during an MSRA internship. Microsoft is a major investor in OpenAI, whose ChatGPT/Codex models are the primary evaluated tools. This financial relationship is not disclosed or discussed as a potential conflict."
     },
     {
       "flag": "No variance for main LLM results",
       "detail": "AutoSD and LLM-Base results on ARHE and Defects4J are presented as single-run point estimates (Tables 1-2). Given LLM output stochasticity, these numbers may vary significantly across runs, but no variance or confidence intervals are reported."
     },
     {
       "flag": "Small sample for subgroup analysis",
       "detail": "The human study (N=20) is split into students (N=14) and professional developers (N=6) for subgroup analysis (Figure 6). N=6 is too small for reliable subgroup conclusions, yet the student-vs-professional comparison is a major discussion point in RQ5."
     },
     {
       "flag": "Unmatched compute comparison",
       "detail": "AutoSD takes 5x longer than LLM-Base (Section 6.2) due to iterative LLM and debugger calls, but the performance comparison does not control for this compute difference. LLM-Base with 5x more samples could potentially outperform AutoSD."
     },
     {
       "flag": "ChatGPT version unspecified",
       "detail": "The default model 'ChatGPT' is described only as 'a sibling model to InstructGPT' without a specific version or API snapshot date. ChatGPT evolved significantly over 2023, making results non-reproducible without version information."
     }
   ],
   "cited_papers": [
     {
       "title": "Impact of Code Language Models on Automated Program Repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "arxiv_id": "2302.05020",
       "relevance": "Large-scale empirical study evaluating multiple LLMs and learning-based APR techniques, providing the baselines (Recoder, InCoder) compared against in this paper."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces Codex and HumanEval benchmark; HumanEval is the basis for the ARHE dataset and Codex is one of the evaluated models."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Describes GPT-4 capabilities and evaluation methodology, referenced for data contamination mitigation context."
     },
     {
       "title": "React: Synergizing reasoning and acting in language models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu"],
       "year": 2022,
       "arxiv_id": "2210.03629",
       "relevance": "Demonstrates LLMs using external tools to improve performance via interleaved reasoning and acting, foundational to AutoSD's tool-use approach."
     },
     {
       "title": "Chain of Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans"],
       "year": 2022,
       "relevance": "Introduces chain-of-thought prompting for LLM reasoning, a key technique underlying AutoSD's hypothesis generation approach."
     },
     {
       "title": "PAL: Program-aided Language Models",
       "authors": ["Luyu Gao", "Aman Madaan", "Shuyan Zhou"],
       "year": 2022,
       "arxiv_id": "2211.10435",
       "relevance": "Shows LLMs can use program execution as external tools to improve reasoning, directly relevant to AutoSD's debugger interaction."
     },
     {
       "title": "Practical Program Repair in the Era of Large Pre-trained Language Models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2022,
       "arxiv_id": "2210.14179",
       "relevance": "Evaluates LLM-based program repair including Codex on Defects4J, providing a reference point for AutoSD's performance comparison."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "relevance": "Demonstrates zero-shot LLM program repair capabilities, directly relevant to AutoSD's zero-shot approach to automated debugging."
     },
     {
       "title": "Training language models to follow instructions with human feedback",
       "authors": ["Long Ouyang", "Jeff Wu", "Xu Jiang"],
       "year": 2022,
       "arxiv_id": "2203.02155",
       "relevance": "Describes RLHF training methodology used for ChatGPT/InstructGPT, the primary model underlying AutoSD."
     },
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational paper on LLM few-shot and zero-shot capabilities, supporting AutoSD's approach of zero-shot scientific debugging."
     },
     {
       "title": "Trust Enhancement Issues in Program Repair",
       "authors": ["Yannic Noller", "Ridwan Shariffdeen", "Xiang Gao", "Abhik Roychoudhury"],
       "year": 2022,
       "doi": "10.1145/3510003.3510040",
       "relevance": "Developer study on APR trust and expectations, finding that explanations including root cause are the most commonly desired output from APR tools."
     },
     {
       "title": "On The Introduction of Automatic Program Repair in Bloomberg",
       "authors": ["Serkan Kirbas", "Etienne Windels", "Olayori McBello"],
       "year": 2021,
       "doi": "10.1109/MS.2021.3071086",
       "relevance": "Industrial deployment of APR at Bloomberg, demonstrating that patches require human review and developers want rationales."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "AutoSD is a practical debugging technique that could be integrated into developer workflows, but requires LLM API access and debugger infrastructure to deploy."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The idea that LLMs can emulate scientific debugging is a novel combination but not contrarian — it confirms the expectation that LLMs can follow structured reasoning processes."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; the paper is about developer tooling for debugging."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict; the paper presents a new technique with balanced evaluation."
     },
     "demo_ability": {
       "score": 0,
       "justification": "Code is not released (only promised), so no one can try the tool immediately."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Microsoft Research Asia is a well-known lab, and the paper uses ChatGPT/OpenAI models which are widely recognized."
     }
   }
 }