ai-research-survey

scan.json (29281B)

---

 {
   "paper": {
     "title": "RGFL: Reasoning Guided Fault Localization for Automated Program Repair Using Large Language Models",
     "authors": [
       "Melika Sepidband",
       "Hamed Taherkhani",
       "Hung Viet Pham",
       "Hadi Hemmati"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.18044",
     "doi": "10.48550/arXiv.2601.18044"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "RGFL introduces per-candidate LLM reasoning into fault localization for automated program repair, improving file-level Hit@1 from 71.4% to 85% and element-level exact match from 36% to 69% on SWE-bench Verified. End-to-end repair resolved rate improved from 51.6% to 58.2%. Embedding-based reranking of reasoning consistently underperformed LLM-based reranking. Counterfactual ablation showed file localization errors have the highest per-case repair impact (50% recovery), while line localization errors are most frequent but hardest to fix in isolation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'We release the source code of our experiments' with a GitHub URL: https://github.com/MelikaSepidband/RGFL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available SWE-bench Verified, SWE-bench Lite, and SWE-bench Java datasets, all referenced with Hugging Face dataset URLs."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or detailed environment setup section is provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link but no step-by-step reproduction instructions, commands to run, or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., '85%', '69%', '58.2%') with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims RGFL 'improves' and 'outperforms' baselines by comparing raw percentages without any statistical significance tests (no p-values, t-tests, or bootstrap tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: 'Hit@1 improves from 71.4% to 85% (19.05% improvement)', 'exact match increases to 69%, which is 91.67% improvement over Agentless', and absolute percentages with both before/after values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification provided for dataset sizes (500, 300, 91 instances). The paper uses standard benchmarks but does not discuss whether these sizes are adequate for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating 40 candidate patches per instance (10 samples × 4 runs), no variance, standard deviation, or spread measure is reported across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against multiple baselines: Agentless, OpenHands, AutoCodeRover, OrcaLoca, and a RAG baseline with BM25 retrieval (Tables 4, 5, 6)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include state-of-the-art systems from the SWE-bench leaderboard: OpenHands (Claude 4 Sonnet), OrcaLoca (Claude 4 Sonnet), Agentless, and TRAE — all very recent systems."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper ablates: (1) file-only vs file+element reasoning (Table 3, Table 6), (2) embedding-based vs LLM-based reranking (Table 2), (3) three different LLMs for reasoning, and (4) counterfactual ablation injecting ground-truth at each localization stage (Section 5.4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Hit@k, Recall@k, MRR for file localization; Exact Match for element localization; Resolved Rate for end-to-end repair."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated via test-suite-based patch validation and metric computation. No human evaluation of the system's outputs is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SWE-bench Verified for both model selection (comparing Gemini 2.5 Pro, Claude 4 Sonnet, o4-mini in RQ1.1) and final result reporting (RQ3). The LLM chosen for downstream stages was selected based on performance on the same test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (Verified, Lite, Java), by localization level (file, element), by LLM (Gemini, Claude, o4-mini), and by error category (file miss, element miss, line miss, repair miss) in the Venn diagram analysis."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.4 conducts detailed error analysis on 209 unresolved instances with a Venn diagram of failure categories, counterfactual ablation, and specific examples where element-level localization hurt performance (Django expressions.py case)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Embedding-based reranking consistently underperformed (Table 2). The paper also reports that providing ground-truth elements can sometimes reduce repair power (9 cases where file-only was better), and o4-mini performed substantially worse than other LLMs."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are directly supported: file-level Hit@1 71.4%→85% and MRR 81.8%→88.8% (Table 2), element-level Exact Match 36%→69% (Table 3), and 12.8% relative repair improvement ((58.2-51.6)/51.6 ≈ 12.8%, Table 6)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('reasoning improves localization') are supported by controlled ablation studies that isolate the file and element localization components while keeping other pipeline stages unchanged. The counterfactual analysis further supports causal attribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The abstract specifies 'Python and Java projects from SWE-bench Verified, Lite, and Java.' Section 6 explicitly acknowledges: 'results may not fully generalize to other ecosystems such as C++, JavaScript/TypeScript, or Rust.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper tests embedding-based vs LLM-based reranking to distinguish reasoning from representation quality. Section 5.4 discusses cases where element localization can hurt. Section 6 discusses model choice dependency and notes OpenHands's improvement 'is very much LLM-dependent.'"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures test-suite passing ('resolved rate') as a proxy for correct repair but does not discuss the gap between passing tests and actual patch correctness. A patch can pass tests while being semantically incorrect."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are referred to by marketing names: 'Gemini 2.5 Pro', 'Claude 4 Sonnet', 'o4-mini'. No API versions, snapshot dates, or exact model IDs are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 provides the full text of all four prompts used: file-level reasoning prompt, file ranking prompt, element-level reasoning prompt, and element ranking prompt. Fill values come from the public SWE-bench dataset, making prompts fully reconstructible."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other generation hyperparameters are reported for any of the LLMs used. Only the number of samples (10 per instance, 4 runs) is stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is described in detail in Section 3 with Figure 2 showing the overview: file retrieval → file-level reasoning → file ranking → element extraction → element-level reasoning → element ranking → line localization → repair → validation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes the datasets and their composition. The methodology explains how Agentless converts the codebase to a tree structure, filters folders, embeds code chunks, and retrieves candidate files before reasoning is applied."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 'Threats to Validity' provides a dedicated section with substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discusses specific threats: language limitation to Python and Java only (not C++/JS/Rust), Gemini 2.5 Pro selection may not generalize to other models, reliance on regression tests without reproduction tests, and repair evaluation limited to SWE-bench Verified only."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 explicitly states what was not tested: other languages (C++, JS/TS, Rust), other model configurations, and repair evaluation on Lite and Java datasets. It also notes that 'Extending repair evaluation to SWE-bench Lite and SWE-bench Java would strengthen the generality.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Source code is released but per-instance experimental data (localization outputs, generated patches, reasoning text) is not mentioned as being released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 describes SWE-bench Verified (500 human-verified instances), Lite (300 instances), and Java (91 instances) with fields and composition."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data source is standard SWE-bench benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline from bug report through file retrieval, reasoning generation, ranking, element localization, line localization, patch generation, and validation is documented across Sections 3 and 4."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgments section or funding information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors are listed as affiliated with York University, Canada. The authors do not evaluate their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed, so funder independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training data cutoff dates are not stated for any of the LLMs used (Gemini 2.5 Pro, Claude 4 Sonnet, o4-mini)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench instances come from public GitHub repositories. The paper does not discuss whether the LLMs may have been trained on the benchmark data or the underlying repositories."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues and patches are publicly available on GitHub and Hugging Face. No discussion of whether models may have seen these during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'The average cost of RGFL per sample is around $4.4, so it incurs a higher computational cost than Agentless (3.7×).' Also mentions ~$400 for Gemini vs ~$2,000 for Claude on Verified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-sample cost ($4.4) and comparative costs are mentioned, but total computational budget (total API spend, GPU hours, wall-clock time) is not explicitly stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating 40 patches per instance across 4 independent runs, no variance or sensitivity analysis across runs is reported."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 states: 'we generate 10 samples per instance, repeated across 4 independent runs, resulting in 40 candidate patches per instance.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Key design choices (top-k=3, 10 samples, 4 runs) appear fixed without search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The LLM selection is justified in RQ1.1: Gemini 2.5 Pro was chosen based on comparable accuracy to Claude 4 Sonnet at substantially lower cost (~$400 vs ~$2,000). The reasoning is transparent."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across datasets, models, and metrics, but no statistical tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate their own RGFL system against baselines. For Agentless, they use their own re-implementation with Gemini 2.5 Pro. For OrcaLoca, they ran Claude 4 Sonnet since trajectory data was unavailable. The self-comparison bias is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 discusses the cost-performance tradeoff: 'RGFL per sample is around $4.4 (3.7× Agentless)... the +12.8% absolute increase in resolved rate provides a compelling justification for the added cost.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench is used without discussion of whether test-suite passing accurately measures repair correctness, or whether SWE-bench instances are representative of real-world bugs."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Internal comparisons control for scaffold by modifying only the localization module. However, cross-system comparisons (OpenHands, OrcaLoca, AutoCodeRover) use different scaffolds and different LLMs simultaneously. The paper notes OpenHands's results are 'LLM-dependent' but does not systematically address the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench issues originate from public GitHub repositories created before the LLMs' training cutoffs. Temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information (e.g., whether the problem statement contains hints about the fix location)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench instances share structural similarities (e.g., multiple issues from the same repository like SymPy or Django)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied (no canary strings, membership inference, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "RGFL improves file-level localization: Hit@1 from 71.4% to 85% and MRR from 81.8% to 88.8% on SWE-bench Verified with Gemini 2.5 Pro.",
       "evidence": "Table 2 reports Hit@k, Recall@k, and MRR for three LLMs with and without reasoning. Consistent improvements across all three models (Gemini, Claude, o4-mini). Section 5.1, RQ1.1.",
       "supported": "strong"
     },
     {
       "claim": "RGFL improves element-level exact match from 36% to 69% on SWE-bench Verified, nearly closing the gap to the theoretical upper bound of 80%.",
       "evidence": "Table 3 shows progressive improvement: Agentless files + Agentless elements = 36%, RGFL files + Agentless elements = 41%, RGFL files + RGFL elements = 69%, with upper bound 80%. Section 5.1, RQ1.2.",
       "supported": "strong"
     },
     {
       "claim": "Embedding-based reranking of reasoning consistently underperforms LLM-based reranking and often degrades localization compared to the Agentless baseline.",
       "evidence": "Table 2 shows Hit@1 drops for all embedding models: Gemini-embedding-001 (64.4% vs 71.4% baseline), Voyage 3.5 (60.8% vs 75.4%), text-embedding-3-small (52% vs 68.4%). Section 5.1.",
       "supported": "strong"
     },
     {
       "claim": "RGFL generalizes across datasets and programming languages, achieving best Hit@3, Recall@3, and MRR on SWE-bench Verified, Lite, and Java.",
       "evidence": "Table 4 reports cross-dataset file localization. Table 5 reports element-level exact match. RGFL achieves best or near-best performance on all three datasets. Section 5.2.",
       "supported": "moderate"
     },
     {
       "claim": "Integrating RGFL into Agentless yields a 12.8% end-to-end repair improvement on SWE-bench Verified (51.6% → 58.2%).",
       "evidence": "Table 6 shows resolved rate: Agentless 51.6%, RGFL file only 55.8%, RGFL file+element 58.2%. However, results are on a single benchmark with no variance reported. Section 5.3.",
       "supported": "moderate"
     },
     {
       "claim": "File localization errors have the highest per-case impact on repair: correcting file misses resolves 50% of affected cases, vs 26% for element and 19% for line.",
       "evidence": "Section 5.4 counterfactual ablation on 209 unresolved instances: GT files resolved 14/28 (50%), GT elements resolved 29/111 (26%), GT lines resolved 33/175 (19%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No statistical significance testing",
       "detail": "All comparative claims ('improves', 'outperforms') are based on raw percentage differences without any statistical tests. Given that results come from stochastic LLM outputs (40 patches per instance), variance could be substantial, and observed differences may not be statistically significant."
     },
     {
       "flag": "No variance reported despite multiple runs",
       "detail": "The paper generates 40 candidate patches per instance (10 samples × 4 independent runs) but never reports variance across these runs. This conceals result stability and makes it impossible to assess whether differences are meaningful."
     },
     {
       "flag": "SWE-bench contamination risk unaddressed",
       "detail": "SWE-bench issues and patches are publicly available on GitHub and Hugging Face. The LLMs used (Gemini 2.5 Pro, Claude 4 Sonnet, o4-mini) were likely trained on data that includes these repositories and possibly the benchmark itself. No training cutoffs are stated and no contamination analysis is performed."
     },
     {
       "flag": "Test set used for model selection",
       "detail": "SWE-bench Verified was used both for selecting the best-performing LLM (Gemini 2.5 Pro over Claude 4 Sonnet and o4-mini in RQ1.1) and for reporting final results. This selection bias inflates reported performance."
     },
     {
       "flag": "Cross-system comparisons confound model and scaffold",
       "detail": "External baseline comparisons use different LLMs (e.g., OpenHands with Claude 4 Sonnet vs RGFL with Gemini 2.5 Pro). The paper acknowledges OpenHands's results are 'LLM-dependent' but still draws comparative conclusions from these confounded comparisons."
     }
   ],
   "cited_papers": [
     {
       "title": "Agentless: Demystifying LLM-based Software Engineering Agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Core baseline and foundation for RGFL's pipeline; state-of-the-art modular APR framework on SWE-bench."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "End-to-end LLM-based APR agent; key baseline achieving top SWE-bench leaderboard performance."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Primary evaluation benchmark for LLM-based program repair; defines the task and evaluation protocol used in this paper."
     },
     {
       "title": "SWE-Agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Pioneering agentic APR system that equips LLMs with structured agent-computer interfaces for repository navigation and editing."
     },
     {
       "title": "AutoCodeRover: Autonomous Program Improvement",
       "authors": ["Yuntong Zhang", "Haifeng Ruan", "Zhiyu Fan", "Abhik Roychoudhury"],
       "year": 2024,
       "relevance": "Pipeline-based APR system using program-structure-aware search APIs; key baseline for localization comparison."
     },
     {
       "title": "OrcaLoca: An LLM Agent Framework for Software Issue Localization",
       "authors": ["Zhongming Yu", "Hejia Zhang", "Yujie Zhao"],
       "year": 2025,
       "arxiv_id": "2502.00350",
       "relevance": "Navigation-based fault localization approach with priority scheduling and distance-aware context pruning; competitive FL baseline."
     },
     {
       "title": "LocAgent: Graph-Guided LLM Agents for Code Localization",
       "authors": ["Zhaoling Chen", "Xiangru Tang", "Gangda Deng"],
       "year": 2025,
       "arxiv_id": "2503.09089",
       "relevance": "Graph-based LLM fault localization using repository-level code graphs; related FL-only approach."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on LLMs for code (Codex); establishes the paradigm of using LLMs for software engineering tasks."
     },
     {
       "title": "Agentic Software Issue Resolution with Large Language Models: A Survey",
       "authors": ["Zhonghao Jiang", "David Lo", "Zhongxin Liu"],
       "year": 2025,
       "arxiv_id": "2512.22256",
       "relevance": "Comprehensive survey of LLM-based APR approaches; provides the taxonomy used to position RGFL."
     },
     {
       "title": "TRAE Agent: An LLM-Based Agent for Software Engineering with Test-Time Scaling",
       "authors": ["Pengfei Gao", "Zhao Tian", "Xiangxin Meng"],
       "year": 2025,
       "arxiv_id": "2507.23370",
       "relevance": "End-to-end LLM agent for software engineering emphasizing test-time scaling and aggressive tool-based exploration."
     },
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM-based APR agent; part of the growing landscape of agentic program repair approaches."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["Patrick Lewis", "Ethan Perez", "Aleksandra Piktus"],
       "year": 2020,
       "relevance": "Foundational RAG paper; RGFL's approach extends RAG-based code retrieval with explicit reasoning."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The method is directly applicable to APR tools and code is released, but requires significant infrastructure (LLM APIs, SWE-bench setup) to deploy."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that explicit reasoning helps LLMs is intuitive; the negative result that embedding-based reranking hurts is mildly surprising."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised; this is a software engineering tool improvement paper."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy; straightforward method comparison with existing tools."
     },
     "demo_ability": {
       "score": 1,
       "justification": "Code is released on GitHub but requires LLM API access, SWE-bench setup, and substantial compute to run."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses well-known benchmarks (SWE-bench) and models (Gemini, Claude), but the authors and lab (York University) are not widely known."
     }
   }
 }