ai-research-survey

scan.json (29691B)

---

 {
   "paper": {
     "title": "T3: Multi-level Tree-based Automatic Program Repair with Large Language Models",
     "authors": ["Quanming Liu", "Xupeng Bu", "Zhichao Yan", "Ru Li"],
     "year": 2025,
     "venue": "IEEE International Joint Conference on Neural Network (IJCNN)",
     "arxiv_id": "2506.21211",
     "doi": "10.1109/IJCNN64981.2025.11228000"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "T3 proposes a multi-level tree-based reasoning framework for automatic program repair that decomposes patch generation into sample retrieval, cause analysis, repair plan generation, and patch generation. On the MODIT dataset, T3 achieves 46.70% (B2Fs) and 28.20% (B2Fm) repair rates with GPT-3.5-turbo, and 48.20%/32.10% with GPT-4o-mini, outperforming CoT baselines by 11.20% and 9.90% respectively. Ablation shows both cause analysis and repair planning contribute, and lower SC accuracy on complex tasks suggests the method explores diverse reasoning paths.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No source code repository URL is provided anywhere in the paper. No GitHub link, Zenodo archive, or supplementary material reference."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the publicly available MODIT dataset [31] including B2Fs and B2Fm subsets [32], which are established public benchmarks of bug fix commits from GitHub."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications are provided. While the paper specifies model versions (gpt-3.5-turbo-0125, gpt-4o-mini-2024-07-18) and mentions the BM25 algorithm, there are no requirements.txt, Docker files, or library version details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The method is described algorithmically but there are no runnable scripts, README, or commands to replicate the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I, II, and III are reported as single point estimates (e.g., '46.70%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are performed. Claims like 'T3 achieves a repair rate of 46.70%, outperforming the CoT baseline by 11.20%' are based solely on comparing raw percentages without any p-values or tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout, e.g., '11.20% improvement over CoT' with raw figures (46.70% vs 35.50% on B2Fs, 28.20% vs 18.30% on B2Fm), providing enough context to assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why the MODIT dataset sizes are sufficient for the claims made, and no power analysis is discussed. The number of programs in each dataset is not even explicitly stated."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Despite generating 30 samples per program with temperature 0.7 (Section IV.B), no standard deviation, variance, or spread measures across runs are reported. All tables show single aggregate numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I compares T3 against seven baselines: NatGen, S-C, S-C+BM25, CoT, Tree-of-Thought, Plan-and-Solve, and Analogical-Reasoning across two models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include methods from 2022-2024: NatGen (2022), Self-Consistency+BM25 (2023), ChatRepair-style CoT variants. These are reasonably contemporary for the APR domain."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table III presents ablation results: 'w/o plan' (37.60%/19.90%), 'w/o cause' (37.30%/21.00%), and full T3 (46.70%/28.20%), demonstrating the contribution of each component."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: repair rate (Table I, proportion of successfully repaired programs) and SC accuracy (Table II, consistency of patch generation)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of repair quality is conducted. Evaluation is entirely automated, based on whether generated patches match ground truth. A single qualitative case study (Table IV) is shown but involves no systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly describe train/test splits or state that results are on a held-out test set. It mentions using the MODIT dataset's B2Fs and B2Fm subsets but does not clarify whether any data was used for tuning decisions."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported separately for B2Fs (smaller code sequences) and B2Fm datasets in all tables, allowing readers to assess performance across task complexity levels."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The case study (Table IV, Section V.E) only shows where competing methods fail and T3 succeeds. No systematic analysis of T3's failure cases or where the approach breaks down is provided."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section V.B honestly reports that T3's SC accuracy on B2Fm (44.68%) is 'inferior relative to other methodologies,' noting a 'compromise in consistency during patch generation' for complex tasks. This is a genuine negative finding."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '11.20% and 9.90% improvement compared to methods based on CoT reasoning,' which matches Table I: B2Fs 46.70% vs 35.50% CoT (=11.20pp), B2Fm 28.20% vs 18.30% (=9.90pp) on GPT-3.5-turbo."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('T3 improves repair accuracy') supported by controlled ablation studies (Table III) that isolate the contribution of cause analysis and repair planning components through single-variable removal."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Automatic Program Repair with Large Language Models' broadly, but experiments use only the MODIT dataset (Java bug fixes from GitHub) with only two OpenAI models. No discussion of generalization boundaries to other languages, bug types, or models."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for the observed improvements. The paper does not discuss whether improvements could stem from increased compute (more reasoning trees = more API calls), prompt engineering advantages, or dataset-specific properties."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures repair rate (exact match of generated patches to correct fixes) and frames results in terms of repair rate. The claims match the granularity of measurements — no broader claims about 'software quality' or 'developer productivity' are made."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Footnote 1 in Section IV.B explicitly states: 'we use the version gpt-3.5-turbo-0125 and gpt-4o-mini-2024-07-18,' providing exact API snapshot versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes the prompting process conceptually (e.g., 'carefully designed prompt template') but never shows the actual prompts used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B reports: temperature = 0.7, 30 generated samples per program. The BM25 retrieval parameter k is described. However, key T3-specific parameters (number of reasoning trees M, top-n values) are not explicitly stated."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four-stage framework is described in detail (Section III): BM25 sample retrieval, Forest of Thinking cause analysis with cross-tree voting, repair plan generation with frequency-based selection, and CoT patch generation. Workflow diagram provided in Fig. 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the MODIT dataset was preprocessed for the experiments. The paper jumps from dataset description ('bug fix commits collected from GitHub') directly to experimental results without documenting any intermediate processing steps."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no limitations section, threats-to-validity section, or any dedicated discussion of the method's limitations. The conclusion does not mention limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper, neither specific nor generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do not show, what settings are excluded, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (generated patches, per-program results, model outputs) is made available. Only aggregated percentages in tables are reported."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The MODIT dataset description is minimal: 'includes two subsets: B2Fs (with smaller code sequences) and B2Fm' with 'bug fix commits collected from GitHub, along with detailed commit logs.' No time period, inclusion/exclusion criteria, or dataset size is stated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. Data source is the MODIT standard benchmark dataset."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "No documentation of the data pipeline from raw dataset to final analysis. The paper does not describe how programs were selected from MODIT, how BM25 retrieval corpora were constructed, or how correctness of patches was verified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. No acknowledgments section mentioning grants or sponsors, despite using commercial API services that cost money."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All four authors list their affiliation as School of Computer and Information Technology, Shanxi University, Taiyuan, China. Email addresses are provided."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Since no funding source is disclosed, independence of the funder cannot be assessed. The paper evaluates OpenAI models, and it is unknown whether OpenAI provided API credits or support."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-3.5-turbo-0125 or GPT-4o-mini-2024-07-18, despite evaluating these models on a publicly available benchmark."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the MODIT dataset (GitHub bug fix commits) could have appeared in the GPT models' training data, which is highly plausible given GitHub is a major training data source."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The MODIT dataset was published in 2019 (Tufano et al.) and is from public GitHub commits, making it very likely to be in GPT-3.5 and GPT-4o training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of automated program repair methods."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or token consumption is reported. The method requires multiple LLM calls per program (multiple reasoning trees, each with multiple CoT expansions, plus voting), making cost a significant practical concern that is completely unaddressed."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. With 30 samples per program across multiple methods and two models, plus the multi-tree reasoning structure, the total API spend is unknown."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis. While temperature 0.7 introduces stochasticity and 30 samples are generated, results are not reported across multiple independent runs with different seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section IV.B states: 'each program in every experimental group is tested with 30 generated samples.' The number of samples per program is explicitly stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Key parameters like the number of reasoning trees M, top-n selection threshold, and BM25 k value appear chosen without justification or search."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for how the final configuration was selected. The values of M (number of trees), n (top-n selection), and k (BM25 examples) are not even explicitly reported, let alone justified through systematic selection."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across methods, models, and datasets without any statistical testing, let alone correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of author-evaluation bias. The authors implement all baseline prompting methods (CoT, Tree-of-Thought, Plan-and-Solve, Analogical-Reasoning) themselves, which risks systematically disadvantaging baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "No comparison at matched compute budgets. T3 uses multiple reasoning trees with multiple CoT expansions each, consuming substantially more API calls than single-chain baselines like CoT. This compute advantage is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the MODIT dataset's exact-match repair metric adequately measures program repair capability. The paper does not consider whether B2Fs/B2Fm are representative of real-world bugs."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The scaffolding difference IS the independent variable being studied. The paper compares different reasoning frameworks (CoT, ToT, Plan-and-Solve, T3) applied to the same models, with the scaffold/prompting strategy as the controlled treatment."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The MODIT dataset contains GitHub commits from before the GPT models' training, meaning the models may have seen both the buggy and fixed versions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the BM25-retrieved similar code examples or the error location information provided to the model constitutes feature leakage beyond what would be available in practice."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the MODIT dataset examples are independent (e.g., multiple commits from the same repository, similar bug patterns that could inflate results)."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, n-gram overlap analysis, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "T3 improves repair rate by 11.20% on B2Fs and 9.90% on B2Fm compared to CoT baseline using GPT-3.5-turbo.",
       "evidence": "Table I: T3 achieves 46.70% (B2Fs) and 28.20% (B2Fm) vs CoT at 35.50% and 18.30% respectively.",
       "supported": "moderate"
     },
     {
       "claim": "T3 maintains robust performance across different model architectures, achieving 48.20% on B2Fs and 32.10% on B2Fm with GPT-4o-mini.",
       "evidence": "Table I shows T3 results on both GPT-3.5-turbo and GPT-4o-mini, with consistent top performance on both.",
       "supported": "moderate"
     },
     {
       "claim": "Both cause analysis and repair plan generation components contribute to T3's performance.",
       "evidence": "Table III ablation: w/o plan = 37.60%/19.90%, w/o cause = 37.30%/21.00%, full T3 = 46.70%/28.20%. Removing either component degrades performance substantially.",
       "supported": "moderate"
     },
     {
       "claim": "T3 employs diverse reasoning pathways for complex tasks, trading consistency for repair efficacy.",
       "evidence": "Table II shows T3 SC accuracy of 44.68% on B2Fm (lowest among methods) while Table I shows highest repair rate (28.20%). Authors interpret this as evidence of path diversity.",
       "supported": "weak"
     },
     {
       "claim": "T3 achieves stable performance with as few as 3 example shots, outperforming other methods in low-sample conditions.",
       "evidence": "Section V.D and Fig. 3 described (but figure data not fully shown in text). Claims T3 stabilizes at 3 shots while Tree-of-Thought needs 5.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All claims of improvement (e.g., 11.20%, 9.90%) are based on comparing raw percentages without any significance testing, confidence intervals, or variance reporting, despite generating 30 stochastic samples per program."
     },
     {
       "flag": "Severe contamination risk unaddressed",
       "detail": "The MODIT dataset (GitHub commits from 2019 or earlier) is almost certainly in the training data of GPT-3.5-turbo and GPT-4o-mini. The models may have memorized the correct fixes, making the benchmark results unreliable. This is not discussed at all."
     },
     {
       "flag": "No code released",
       "detail": "Neither source code nor prompts are provided, making independent replication impossible. The exact prompts used for each method are critical to the results but are not shared."
     },
     {
       "flag": "Unfair compute comparison",
       "detail": "T3 uses multiple reasoning trees (M trees × N expansions each) for both cause analysis and repair planning, consuming far more API calls than single-chain baselines. No cost normalization or compute-matched comparison is provided."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or scope boundaries discussion. This is a significant omission for a methods paper making comparative claims."
     },
     {
       "flag": "Key parameters unreported",
       "detail": "The number of reasoning trees M and top-n selection threshold — core design parameters of T3 — are never explicitly stated as concrete values used in experiments, only described abstractly."
     },
     {
       "flag": "Single case study as qualitative evidence",
       "detail": "Table IV shows one cherry-picked example where T3 succeeds and all competitors fail or partially fail. No systematic qualitative analysis or failure cases of T3 are shown."
     }
   ],
   "cited_papers": [
     {
       "title": "RAP-Gen: Retrieval-augmented Patch Generation with CodeT5 for Automatic Program Repair",
       "authors": ["W. Wang", "Y. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2023,
       "relevance": "Retrieval-augmented APR using CodeT5, directly relevant as a fine-tuning-based LLM repair method."
     },
     {
       "title": "Less Training, More Repairing Please: Revisiting Automated Program Repair via Zero-shot Learning",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2022,
       "relevance": "ALPHA — zero-shot cloze-style APR using LLMs, a key baseline approach for LLM-based program repair."
     },
     {
       "title": "GAMMA: Revisiting Template-based Automated Program Repair via Mask Prediction",
       "authors": ["Q. Zhang", "C. Fang", "T. Zhang", "B. Yu", "W. Sun", "Z. Chen"],
       "year": 2023,
       "relevance": "Template-based APR using mask prediction with LLMs, representing the cloze-style repair paradigm."
     },
     {
       "title": "Automated Program Repair via Conversation: Fixing 162 out of 337 Bugs for $0.42 Each Using ChatGPT",
       "authors": ["C. S. Xia", "L. Zhang"],
       "year": 2024,
       "relevance": "ChatRepair — conversation-based APR using ChatGPT with iterative feedback, a major LLM-based repair approach."
     },
     {
       "title": "CodeT5: Identifier-aware Unified Pre-trained Encoder-Decoder Models for Code Understanding and Generation",
       "authors": ["Y. Wang", "W. Wang", "S. Joty", "S. C. Hoi"],
       "year": 2021,
       "relevance": "Foundation code LLM used in multiple APR systems, relevant to understanding LLM capabilities for code tasks."
     },
     {
       "title": "CodeBERT: A Pre-trained Model for Programming and Natural Languages",
       "authors": ["Z. Feng", "D. Guo", "D. Tang", "N. Duan"],
       "year": 2020,
       "relevance": "Pre-trained code representation model used as foundation for multiple code understanding and repair approaches."
     },
     {
       "title": "Chain-of-thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["J. Wei", "X. Wang", "D. Schuurmans"],
       "year": 2022,
       "relevance": "Foundational CoT prompting technique that the paper builds upon and compares against for APR tasks."
     },
     {
       "title": "Self-consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["X. Wang", "J. Wei", "D. Schuurmans", "Q. V. Le"],
       "year": 2023,
       "relevance": "Self-consistency mechanism used in T3's voting/selection strategy for error causes and repair plans."
     },
     {
       "title": "Better Patching Using LLM Prompting, via Self-Consistency",
       "authors": ["T. Ahmed", "P. Devanbu"],
       "year": 2023,
       "relevance": "Directly relevant prior work applying self-consistency to LLM-based patching, source of the MODIT dataset and S-C baselines used in this paper."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Codex paper — foundational evaluation of LLMs for code generation, establishing key evaluation paradigms."
     },
     {
       "title": "A Systematic Literature Review on Large Language Models for Automated Program Repair",
       "authors": ["Q. Zhang", "C. Fang", "Y. Xie"],
       "year": 2024,
       "arxiv_id": "2405.01466",
       "relevance": "Comprehensive survey of LLM-based APR methods, directly relevant to the survey scope."
     },
     {
       "title": "DEAR: A Novel Deep Learning-based Approach for Automated Program Repair",
       "authors": ["Y. Li", "S. Wang", "T. N. Nguyen"],
       "year": 2022,
       "relevance": "Deep learning-based APR approach representing the learning-based repair paradigm."
     },
     {
       "title": "NatGen: Generative Pre-training by 'Naturalizing' Source Code",
       "authors": ["S. Chakraborty", "T. Ahmed", "Y. Ding", "P. T. Devanbu", "B. Ray"],
       "year": 2022,
       "relevance": "Generative pre-training approach for code used as a baseline in the paper's experiments."
     },
     {
       "title": "Plan-and-Solve Prompting: Improving Zero-shot Chain-of-Thought Reasoning by Large Language Models",
       "authors": ["L. Wang", "W. Xu", "Y. Lan"],
       "year": 2023,
       "relevance": "Advanced CoT variant used as both a baseline and component in T3's reasoning framework."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "The multi-tree reasoning approach is interesting but no code is released, making it not immediately usable by practitioners."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that structured multi-path reasoning improves LLM performance on complex tasks."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or AI risk concerns raised; the paper is about improving bug-fixing accuracy."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or provocative claims; straightforward methods paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or tool released; results cannot be reproduced or tried."
     },
     "brand_recognition": {
       "score": 0,
       "justification": "Authors are from Shanxi University with no major brand recognition in the LLM/APR community."
     }
   }
 }