ai-research-survey

scan.json (30200B)

---

 {
   "paper": {
     "title": "Enhancing Automated Program Repair via Faulty Token Localization and Quality-Aware Patch Refinement",
     "authors": [
       "Jiaolong Kong",
       "Xiaofei Xie",
       "Yiheng Xiong",
       "Yuekun Wang",
       "Jian Wang"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.18001",
     "doi": "10.48550/arXiv.2511.18001"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "TokenRepair combines token-level uncertainty-based fault localization (internal reflection) with quality-aware patch refinement (external feedback) for automated program repair. Across five 7B-class LLMs, it achieves 88 correct fixes on Defects4J 1.2 and 139 on HumanEval-Java, outperforming the best baseline by 8.2%–34.9% on Defects4J and 3.3%–16.1% on HumanEval-Java. Ablation confirms uncertainty-guided localization is the most impactful component, with removal causing up to 20.6% performance degradation.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'we have made our patches open-source for public evaluation' but provides no repository URL, download link, or archive reference anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses two standard public benchmarks: Defects4J 1.2 (154 single-hunk bugs) and HumanEval-Java (163 bugs), both publicly available and unmodified."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper lists model names with HuggingFace references and some hyperparameters (temperature=1, TopK=3, α=0.5) but provides no requirements.txt, Dockerfile, library versions, or environment setup instructions sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are described or referenced in the paper."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1–6 are point estimates (counts of correct fixes, accuracy values, percentages). No confidence intervals or error bars are reported despite the stochastic nature of the method (temperature=1)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims TokenRepair 'outperforms' and 'surpasses' baselines based solely on comparing raw numbers (e.g., 63 vs 51 correct fixes). No statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Percentage improvements with baseline context are reported throughout: '8.2% to 34.9% across all models on Defects4J 1.2' and '3.3% to 16.1% on HumanEval-Java,' with both baseline and improved counts given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks contain 154 and 163 bugs respectively. No justification is given for why these sizes are sufficient for the claims made, nor any power analysis."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single experimental runs. No standard deviation, variance across seeds, or spread measures are reported, despite using stochastic sampling (temperature=1) where results can vary significantly across runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: Base Sampling, CoT-Decoding, and ChatRepair, each described in Section 4.1.3 with matched configurations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ChatRepair (2024), CoT-Decoding (2024 NeurIPS), and Base Sampling (2025) are all recent and represent the state of the art in LLM-based APR."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "RQ3 (Section 5.3) presents a thorough ablation with three variants: w/o Majority, w/o Localize, and w/o Quality, measuring the contribution of each component across all models and both benchmarks (Table 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: #Plausible (test-passing patches), #Correct (manually verified correct patches), and #Generate (average patches per correct fix, measuring efficiency)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 describes manual patch verification: 'inviting three researchers in SE field to check respectively, each dedicating over 10 hours to manually validate the patches. They then discuss the patches where validation answers were inconsistent, ultimately reaching a consensus.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters (α, TopK, n, m) are tuned via grid search on the same Defects4J and HumanEval-Java benchmarks used for final evaluation (Tables 1, 6). The best configuration per model in Table 4 matches the best from Table 6, indicating test-set hyperparameter optimization."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (5 models) and per benchmark (2 benchmarks) in Table 4. Venn diagrams (Fig. 4) show unique fixes per method. Table 6 provides per-configuration breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly mentions DeepSeek's exception on HumanEval-Java attributed to low localization accuracy, but provides no systematic failure case analysis showing specific bugs TokenRepair fails on or qualitative examples of failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "RQ4 reports that m=9 'never achieves the best performance across any model or benchmark' and explains why. The DeepSeek exception on HumanEval-Java (98 vs 99 for Base Sampling) is explicitly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 88 bugs on Defects4J 1.2 and 139 on HumanEval-Java match Venn diagram aggregates in Fig. 4. Improvement ranges of 8.2%–34.9% on Defects4J and 3.3%–16.1% on HumanEval-Java are verifiable from Table 4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('X improves Y') are supported by controlled ablation studies (RQ3) that systematically remove individual components while holding others constant, demonstrating their contribution."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper tests only Java benchmarks with five 7B-class open-source models on single-hunk bugs, but the title ('Enhancing Automated Program Repair') and abstract ('state-of-the-art repair performance') make unbounded claims without qualifying to these specific settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 discusses threats related to manual verification and LLM non-determinism but does not consider alternative explanations for why the method works (e.g., whether improvements stem from simply exploring more diverse patches rather than targeted token refinement)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures #Correct (manually verified semantic equivalence to ground truth) and #Plausible (passes test suite) and clearly defines both. Claims match the granularity of measurements without overclaiming broader outcomes."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model identifiers are given: Qwen2.5-Coder-7B-Instruct, Llama-3.1-8B-Instruct, DeepSeek-Coder-6.7b-Instruct, DeepSeek-Coder-7b-Instruct-V1.5, CodeGemma-7b-it, with corresponding HuggingFace URLs in references."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompt contents conceptually ('buggy code and failure information,' 'ConstructPrompt') but never provides actual prompt text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.1 reports: temperature=1, TopK=3, α=0.5, n∈{2,5}, m∈{3,6,9}, budget=50 patches. RQ4 evaluates sensitivity to n and m."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The full scaffolding is described in detail: Algorithm 1 gives the complete procedure, Section 3 details the internal reflection loop (uncertainty calculation, CoT-decoding), external feedback loop (quality measurement, patch filtering), and their interaction."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 states: 'We evaluate TokenRepair under the single-hunk fix scenario, and adopt the benchmark construction process in prior research, where the location of the buggy hunk is provided based on the ground truth.' Targets 154 single-hunk bugs in Defects4J 1.2 and all 163 in HumanEval-Java."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' provides substantive discussion of manual verification challenges, experimental non-determinism, and floating-point precision issues."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 discusses study-specific threats: manual patch verification requiring three researchers with 10+ hours each, non-deterministic LLM inference, and floating-point precision accumulation in uncertainty calculations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show — e.g., no mention that results are limited to Java, single-hunk bugs, or 7B-class models, and no discussion of what settings or scenarios were not tested."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Although patches are claimed to be open-sourced, no URL is provided. Raw experimental data (uncertainty scores, intermediate patches, per-bug outcomes) are not available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 describes using Defects4J (154 single-hunk bugs from 17 projects) and HumanEval-Java (163 buggy-fixed code pairs), following established benchmark construction from prior research."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; data sources are standard public benchmarks (Defects4J and HumanEval-Java)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 documents the full pipeline from buggy code input through patch generation, evaluation, refinement, quality filtering, and output. Section 4.1.4 defines how metrics are computed from patch outcomes."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding section, acknowledgments, or grant references appear anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All five authors are listed as affiliated with Singapore Management University. The paper evaluates open-source models, not SMU products, so no product-affiliation conflict exists."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Cannot assess funder independence since funding is not disclosed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the five models used (Qwen2.5-Coder, Llama-3.1, DeepSeek-Coder, CodeGemma)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Defects4J or HumanEval-Java examples appeared in the training data of any model, despite all models being trained well after these benchmarks were published."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J was published in 2014 and HumanEval in 2021, both long before the 2024 models were trained. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The manual patch verification is performed by researchers as evaluators, not as study subjects."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports #Generate (average patches per correct fix) as an efficiency proxy but does not report actual inference cost in tokens, dollars, GPU time, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Patch generation budget is capped at 50 per bug, but no GPU hours, total wall-clock time, or hardware specifications are reported for the experiments across 5 models × 2 benchmarks × multiple configurations."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis, despite using stochastic sampling at temperature=1 where results can vary substantially across runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state how many experimental runs produced the reported results. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "RQ1 evaluates α∈{0.2,0.5,0.8} × TopK∈{1,2,3,4,5} and RQ4 evaluates n∈{2,5} × m∈{3,6,9}. All configurations and their results are enumerated in Tables 1 and 6."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Selection of α=0.5 and TopK=3 is justified in RQ1 based on average accuracy analysis across models. For n and m, all configurations are reported in Table 6, and the rationale for selecting best configs is discussed."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes dozens of comparative claims across 5 models × 2 benchmarks × 4 methods without any statistical tests, let alone multiple comparison correction."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors re-implement baselines (ChatRepair, Base Sampling, CoT-Decoding) to match their experimental settings but do not acknowledge the systematic bias identified by Lucic et al. (2018) in author-implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "All methods are given the same patch budget (50), providing a fair comparison, but no performance-vs-compute curves are shown. The relationship between budget consumption and repair success is not analyzed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether Defects4J or HumanEval-Java actually measure the APR capabilities the paper claims to evaluate. No analysis of construct validity or comparison with alternative benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The comparison is well-controlled: all four methods (Base Sampling, CoT-Decoding, ChatRepair, TokenRepair) are evaluated on the same five models with matched budgets. The repair strategy (scaffold) is the variable under test, properly isolated from the model variable."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Defects4J (2014) and HumanEval (2021) predate all five models (2024). Solutions and discussion of these benchmarks are widely available online. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The evaluation provides ground-truth buggy hunk location to the model, acknowledged as standard practice but not discussed as a form of feature leakage that inflates repair rates compared to real-world scenarios."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data for the five models includes Defects4J projects or HumanEval solutions, nor any analysis of non-independence between train and test data."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "TokenRepair achieves 88 correct fixes on Defects4J 1.2 and 139 on HumanEval-Java when aggregating across all five models.",
       "evidence": "Venn diagrams in Fig. 4 show cumulative correct fixes across all five models: 88 on Defects4J and 139 on HumanEval-Java, compared to 82 and 131 for the next best (ChatRepair).",
       "supported": "strong"
     },
     {
       "claim": "TokenRepair achieves improvements of 8.2%–34.9% over the best baseline per model on Defects4J 1.2.",
       "evidence": "Table 4 shows per-model correct fixes: Qwen 51→63 (23.5%), Llama 49→53 (8.2%), DeepSeek 46→53 (15.2%), CodeGemma 43→58 (34.9%), DeepSeek-V1.5 43→54 (25.6%).",
       "supported": "moderate"
     },
     {
       "claim": "Token-level uncertainty localization achieves average accuracy ranging from 0.589 to 0.695 for identifying faulty tokens.",
       "evidence": "Table 1 presents Top-K accuracy across five models, two benchmarks, and three decay factors. Average accuracy at optimal α=0.5 ranges from 0.589 (DeepSeek on HumanEval-Java) to 0.695 (Llama and DeepSeek on Defects4J).",
       "supported": "moderate"
     },
     {
       "claim": "Uncertainty-guided token localization is the most impactful component, with removal causing up to 20.6% performance degradation.",
       "evidence": "Table 5 ablation: removing localization drops Qwen on Defects4J from 63 to 50 correct fixes (20.6%), the largest drop among all ablation variants across all models.",
       "supported": "moderate"
     },
     {
       "claim": "Decreasing uncertainty across repair iterations correlates with correct fixes.",
       "evidence": "Table 3 shows plausible paths have 54.1%–80.5% decreasing uncertainty transitions, while incorrect paths show near-equal distributions (40.6%–51.9% decreasing).",
       "supported": "weak"
     },
     {
       "claim": "TokenRepair uniquely fixes 7 bugs on Defects4J and 2 bugs on HumanEval-Java that no baseline can fix.",
       "evidence": "Venn diagrams in Fig. 4 show 7 unique fixes on Defects4J and 2 on HumanEval-Java.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No variance or seed analysis for stochastic method",
       "detail": "The method uses temperature=1 sampling, making results inherently non-deterministic. Yet all results are reported as single-run point estimates without any variance, standard deviation, or multiple-seed analysis. The paper itself acknowledges in Section 7 that 'the non-deterministic nature of LLM inference' is a threat, but does not mitigate it."
     },
     {
       "flag": "Hyperparameter optimization on test set",
       "detail": "The best (n, m) configuration per model reported in Table 4 corresponds to the best result from the full grid search in Table 6, conducted on the same test benchmarks. No separate validation set is used for hyperparameter selection."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All superiority claims are based on comparing raw counts (e.g., 63 vs 51 correct fixes). With stochastic sampling and small benchmark sizes (154 and 163 bugs), the observed differences may not be statistically significant."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "All five models (2024) were trained well after Defects4J (2014) and HumanEval (2021) were published. Solutions to these benchmarks are widely available online. No contamination analysis is performed."
     },
     {
       "flag": "Self-authored baseline",
       "detail": "First author Kong is also a co-author of ContrastRepair (cited as [18]), one of the related works. All baselines are re-implemented by the authors to match their experimental settings, introducing potential implementation bias not acknowledged in the paper."
     },
     {
       "flag": "Limited model diversity",
       "detail": "All five models are 7B-class open-source models. No larger models (70B+), proprietary models (GPT-4, Claude), or models with different architectures are tested, limiting generalizability of the findings."
     }
   ],
   "cited_papers": [
     {
       "title": "RepairAgent: An Autonomous, LLM-Based Agent for Program Repair",
       "authors": ["Islem Bouzenia", "Premkumar Devanbu", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2403.17134",
       "relevance": "Autonomous LLM-based agent for program repair using dynamic prompting and planning — directly relevant to agentic APR approaches."
     },
     {
       "title": "ContrastRepair: Enhancing Conversation-Based Automated Program Repair via Contrastive Test Case Pairs",
       "authors": ["Jiaolong Kong", "Xiaofei Xie", "Mingfei Cheng", "Shangqing Liu", "Xiaoning Du", "Qi Guo"],
       "year": 2025,
       "doi": "10.1145/3719345",
       "relevance": "Conversation-based APR using contrastive test case pairs — pioneering work in multi-turn LLM repair."
     },
     {
       "title": "Demystifying Memorization in LLM-Based Program Repair via a General Hypothesis Testing Framework",
       "authors": ["Jiaolong Kong", "Xiaofei Xie", "Shangqing Liu"],
       "year": 2025,
       "relevance": "Investigates memorization and contamination issues in LLM-based APR — directly relevant to benchmark contamination concerns."
     },
     {
       "title": "Impact of code language models on automated program repair",
       "authors": ["Nan Jiang", "Kevin Liu", "Thibaud Lutellier", "Lin Tan"],
       "year": 2023,
       "relevance": "Evaluates code LLMs for APR and introduces HumanEval-Java benchmark used in this paper."
     },
     {
       "title": "Automated program repair in the era of large pre-trained language models",
       "authors": ["Chunqiu Steven Xia", "Yuxiang Wei", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Foundational study on LLM-based APR showing pre-trained models can surpass traditional APR techniques."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-Shot Learning",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2022,
       "doi": "10.1145/3540250.3549101",
       "relevance": "Zero-shot LLM approach to APR demonstrating repair without task-specific fine-tuning."
     },
     {
       "title": "Automated program repair via conversation: Fixing 162 out of 337 bugs for $0.42 each using ChatGPT",
       "authors": ["Chunqiu Steven Xia", "Lingming Zhang"],
       "year": 2024,
       "relevance": "ChatRepair — the primary baseline in this paper, pioneering conversation-based iterative APR with LLMs."
     },
     {
       "title": "Chain-of-thought reasoning without prompting",
       "authors": ["Xuezhi Wang", "Denny Zhou"],
       "year": 2024,
       "relevance": "CoT-Decoding method used as a baseline — explores alternative decoding paths at inference time."
     },
     {
       "title": "Calibration and correctness of language models for code",
       "authors": ["Claudio Spiess", "David Gros", "Kunal Suresh Pai", "Michael Pradel"],
       "year": 2024,
       "arxiv_id": "2402.02047",
       "relevance": "Shows correlation between LLM uncertainty/entropy and code token correctness — foundational finding that TokenRepair builds upon."
     },
     {
       "title": "CigaR: Cost-efficient Program Repair with LLMs",
       "authors": ["Dávid Hidvégi", "Khashayar Etemadi", "Sofia Bobadilla", "Martin Monperrus"],
       "year": 2024,
       "arxiv_id": "2402.06598",
       "relevance": "Token-efficient LLM-based APR approach addressing computational cost of conversation-based repair."
     },
     {
       "title": "RepairLLaMA: Efficient Representations and Fine-Tuned Adapters for Program Repair",
       "authors": ["André Silva", "Sen Fang", "Martin Monperrus"],
       "year": 2023,
       "arxiv_id": "2312.15698",
       "relevance": "Fine-tuning approach for APR using LoRA adapters on CodeLlama — represents the fine-tuning paradigm in LLM-based repair."
     },
     {
       "title": "Uncertainty-guided chain-of-thought for code generation with LLMs",
       "authors": ["Yuqi Zhu", "Ge Li", "Xue Jiang", "Jia Li", "Hong Mei", "Zhi Jin", "Yihong Dong"],
       "year": 2025,
       "arxiv_id": "2503.15341",
       "relevance": "Uses first-token uncertainty as a quality indicator for code generation — directly informs TokenRepair's trace quality measurement design."
     },
     {
       "title": "Can OpenAI's Codex fix bugs? An evaluation on QuixBugs",
       "authors": ["Julian Aron Prenner", "Hlib Babii", "Romain Robbes"],
       "year": 2022,
       "relevance": "Early evaluation of LLM (Codex) for automated program repair on QuixBugs benchmark."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Token-level uncertainty refinement for APR is a usable technique but requires access to model logits (token probabilities), limiting it to open-weight models."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The internal reflection + external feedback framing is novel but the core idea (uncertainty-guided refinement) is an incremental extension of existing work."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or security concerns raised — purely a program repair improvement."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code repository URL, demo, or installable tool is provided despite claiming patches are open-sourced."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "From Singapore Management University — not a high-profile AI lab. Uses smaller open-source models, not flagship products."
     }
   }
 }