ai-research-survey

scan.json (29984B)

---

 {
   "paper": {
     "title": "Monte Carlo Tree Search for Execution-Guided Program Repair with Large Language Models",
     "authors": ["Yixuan Liang"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.00129"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CodePilot combines Qwen3-8B with Monte Carlo Tree Search for automated GitHub issue resolution, achieving 24.67% resolve rate on SWE-bench Lite (300 instances). Ablation studies show MCTS contributes the largest improvement (+4.34pp), followed by thinking mode (+3.67pp) and self-refinement (+3.00pp). However, all results appear to be single-run with no statistical tests or uncertainty quantification, and the paper provides no code, no limitations section, and claims contamination-free evaluation without any contamination analysis.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or download link is provided anywhere in the paper. The framework 'CodePilot' is described but no source code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates on SWE-bench Lite, a publicly available benchmark. The fine-tuning dataset is described as 'historical GitHub issues with verified patches' but is not released separately."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA A100 GPUs' and 'vLLM inference' (Section VII.A) but provides no requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology sections describe the approach conceptually but do not include commands, scripts, or a reproduction guide."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables I and II are reported as point estimates (e.g., '24.67%') with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims CodePilot 'significantly outperforming existing open-weight baselines' but no statistical significance test (p-value, t-test, bootstrap, etc.) is reported. Differences are assessed by comparing raw percentages only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table I provides baseline context (e.g., Agentless Qwen3-8B at 19.33% vs CodePilot at 24.67%, a +5.34pp improvement). Table II reports deltas for each ablation (e.g., -4.34% for w/o MCTS). These provide sufficient context to assess effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses SWE-bench Lite (300 instances) without justifying this sample size or discussing whether 300 instances provide sufficient statistical power for the claimed comparisons."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measure is reported. All results appear to be from single runs with no indication of result stability across seeds or repetitions."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table I compares against Direct Generation (CodeLlama-7B), Agentless (CodeLlama-7B, DeepSeek-7B, Qwen2.5-7B, Qwen3-8B), and SWE-agent (Qwen3-8B)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include recent models (Qwen3-8B, Qwen2.5-7B) and recent frameworks (Agentless, SWE-agent). However, only open-weight baselines of similar scale are compared — no proprietary or larger-scale baselines are included."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table II presents an ablation study removing MCTS (-4.34%), thinking mode (-3.67%), self-refinement (-3.00%), and hybrid retrieval (-2.34%), plus a direct generation baseline (-10.67%)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both Resolve Rate and Apply Rate (Tables I and II). Section VI also defines CodeBLEU and Localization Accuracy metrics, though these are not reported in the results tables."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via test suite pass/fail on SWE-bench Lite. For a program repair system, human review of patch quality (correctness, readability, side effects) would be relevant."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SWE-bench Lite is a standard evaluation benchmark separate from the fine-tuning data, which is described as 'historical GitHub issues with verified patches' (Section IV.A). The benchmark serves as a held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregate resolve rate and apply rate are reported. No per-repository, per-difficulty, or per-category breakdown is provided. SWE-bench Lite spans multiple repositories with varying difficulty, but this variation is hidden behind a single number."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section III.B states 'our analysis shows incorrect localization accounts for one-third of failures,' providing a quantitative failure analysis. However, no qualitative examples of specific failures are shown."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": false,
         "justification": "Every component in the ablation study (Table II) shows a positive contribution. No approaches that were tried and abandoned, configurations that failed, or unexpected negative outcomes are discussed."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims CodePilot 'significantly outperforming existing open-weight baselines' but no statistical significance test is performed — the comparison is based on raw percentage differences only. The word 'significantly' implies a tested statistical claim that is not substantiated."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table II) makes causal claims by removing individual components (MCTS, thinking mode, self-refinement, hybrid retrieval) and measuring the impact. This controlled single-variable manipulation is an adequate design for causal attribution, though single-run results weaken confidence."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Program Repair with Large Language Models' broadly, and the abstract discusses 'automated GitHub issue resolution' in general terms. However, results are only on SWE-bench Lite (300 Python instances) with a single model (Qwen3-8B). No bounding to this specific setting is stated."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the results are discussed. The improvement could be partly due to the fine-tuning data, the specific Qwen3 model capabilities, or SWE-bench Lite characteristics rather than MCTS alone. None of these confounds are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures resolve rate on SWE-bench Lite but frames CodePilot as solving 'automated software engineering' and 'GitHub issue resolution' broadly. The gap between benchmark performance and real-world issue resolution capability is not acknowledged."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The main model is specified as 'Qwen3-8B-Instruct' (Section VII.A). Baseline models include 'CodeLlama-7B', 'DeepSeek-7B', 'Qwen2.5-7B'. These include model family, size, and variant (Instruct). Note: Section IV mentions 'Qwen3-32B-Instruct' inconsistently."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided. The paper describes prompt functionality conceptually (e.g., presenting functions in dependency order, using thinking mode) but never shows the actual prompts sent to the model."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section VII.A reports: 'T = 0.6, top-p = 0.95. MCTS parameters: c = 1.4, K = 3, N = 16. LoRA uses rank r = 16.' Key inference and search parameters are specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full pipeline in detail: hierarchical fault localization (Section III.B), MCTS-guided patch synthesis with UCT selection (Section III.C), execution-driven self-refinement loop (Section III.D), and confidence calibration (Section III.E). Workflow diagrams are provided in Figs. 1-3."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section V describes repository parsing via AST, semantic-aware code chunking with overlap strategy, issue text normalization including HTML cleaning and stack trace extraction, and context window packing with importance weighting."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations, threats to validity, or discussion section is present. The paper goes directly from results (Section VII) to conclusion (Section VIII) without discussing limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. The conclusion mentions 'future work on multi-file modifications' but does not discuss specific threats to the current study's validity."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clarify what the results do NOT show, what settings were excluded, or what claims the authors are not making. The conclusion mentions future work but not current scope limitations."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-instance results, generated patches, execution logs) is released. Only aggregate statistics in Tables I and II are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The fine-tuning dataset is described as 'historical GitHub issues with verified patches' (Section IV.A) but no details are provided about which repositories, time period, size, or selection criteria were used for this dataset."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The evaluation uses SWE-bench Lite, a standard benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While Section V documents the preprocessing pipeline for inference inputs, the fine-tuning data pipeline is undocumented — no information about how many training instances were collected, what filtering was applied, or how ground-truth patches were verified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper. Whether the work is funded or unfunded is not stated."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is listed as 'Illinois Institute of Technology, Chicago, USA.' No conflict with the evaluated model vendor (Alibaba/Qwen) is apparent."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funding is disclosed. The work appears to be unfunded student research by a solo author at a university."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The training data cutoff for Qwen3-8B-Instruct is not stated. The paper additionally fine-tunes the model but does not specify the base model's training data coverage."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether SWE-bench Lite instances appear in Qwen3's pre-training data. No canary strings, membership inference, or temporal analysis is performed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "SWE-bench was published in 2023 (arXiv:2310.06770) and is widely discussed online. Qwen3 could have been trained on its data. The paper twice claims 'contamination-free' (abstract, conclusion) but provides zero evidence to support this claim."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost, latency, or tokens consumed are reported. The MCTS approach with multiple expansion-simulation cycles per instance likely has significant compute cost, but this is not quantified."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA A100 GPUs' (Section VII.A) but does not state total GPU hours, wall-clock time, or compute budget for either fine-tuning or inference."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be from a single run, despite MCTS having stochastic elements that would make results seed-sensitive."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented as single point estimates without indicating how many repetitions produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "MCTS parameters (c=1.4, K=3, N=16) and LoRA rank (r=16) are reported but no search budget, search method, or number of configurations tried is stated. It is unclear how these values were selected."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "No explanation of how the reported configuration was selected. The paper does not describe whether a validation set was used for tuning or how many alternatives were tried."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (Direct Generation, and presumably re-run Agentless and SWE-agent) without acknowledging that author-implemented baselines can systematically underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "CodePilot uses MCTS with multiple expansion-simulation cycles, LoRA fine-tuning, and PPO — likely using substantially more compute than simple baselines like Direct Generation or Agentless. This compute difference is not discussed or controlled for."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench Lite adequately measures the claimed capability of 'automated software engineering' or 'GitHub issue resolution.' The benchmark's limitations and construct validity are not examined."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Table I compares CodePilot (MCTS + hierarchical localization + self-refinement) against Agentless and SWE-agent, which use entirely different scaffolds. The performance difference is attributed to the approach without controlling for the scaffold confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether SWE-bench Lite issues and their solutions existed before Qwen3's training cutoff. SWE-bench issues come from real GitHub repositories and may have been part of training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The paper's retrieval-augmented approach provides the model with selected code context, but whether this mirrors realistic deployment conditions is not analyzed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the fine-tuning data ('historical GitHub issues with verified patches') shares repositories or structural similarities with SWE-bench Lite instances."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is used despite the paper twice claiming 'contamination-free' evaluation. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipeline is described."
       }
     }
   },
   "claims": [
     {
       "claim": "CodePilot achieves 24.67% resolve rate on SWE-bench Lite, outperforming existing open-weight baselines.",
       "evidence": "Table I shows CodePilot at 24.67% resolve rate vs Agentless (Qwen3-8B) at 19.33% and SWE-agent (Qwen3-8B) at 16.67%.",
       "supported": "moderate"
     },
     {
       "claim": "MCTS contributes the largest individual improvement (+4.34pp) among all components.",
       "evidence": "Table II ablation shows removing MCTS drops resolve rate from 24.67% to 20.33%.",
       "supported": "moderate"
     },
     {
       "claim": "Thinking mode contributes +3.67pp improvement.",
       "evidence": "Table II shows removing thinking mode drops resolve rate from 24.67% to 21.00%.",
       "supported": "moderate"
     },
     {
       "claim": "Self-refinement contributes +3.00pp improvement.",
       "evidence": "Table II shows removing self-refinement drops resolve rate from 24.67% to 21.67%.",
       "supported": "moderate"
     },
     {
       "claim": "Incorrect localization accounts for one-third of failures.",
       "evidence": "Section III.B states 'our analysis shows incorrect localization accounts for one-third of failures' but no supporting data or breakdown is provided.",
       "supported": "weak"
     },
     {
       "claim": "CodePilot 'significantly outperforms' existing open-weight baselines.",
       "evidence": "Table I shows percentage improvements but no statistical significance tests are performed. The word 'significantly' is used in the statistical sense without any test.",
       "supported": "weak"
     },
     {
       "claim": "The evaluation is on 'contamination-free software engineering benchmarks.'",
       "evidence": "Abstract and Section VIII assert contamination-free evaluation but provide no evidence — no training cutoff analysis, no overlap detection, no contamination testing.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results in Tables I and II appear to be single-run point estimates with no uncertainty quantification. MCTS is inherently stochastic, making seed sensitivity analysis especially important, yet none is reported."
     },
     {
       "flag": "'Significantly outperforms' without statistical test",
       "detail": "The abstract uses the word 'significantly' — a term with specific statistical meaning — but no significance test is performed. The claimed improvements are assessed only by comparing raw percentages."
     },
     {
       "flag": "Unsubstantiated contamination-free claim",
       "detail": "The paper twice claims 'contamination-free' evaluation (abstract, conclusion) without any contamination analysis, training cutoff discussion, or leakage detection. SWE-bench has been public since 2023 and Qwen3 could have trained on it."
     },
     {
       "flag": "No code or reproducibility artifacts released",
       "detail": "Despite proposing a complete framework (CodePilot), no source code, scripts, or reproduction instructions are provided. The results cannot be independently verified."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats to validity, or discussion section. It goes directly from results to a short conclusion, providing no self-critical assessment."
     },
     {
       "flag": "Model size inconsistency",
       "detail": "Section III.A describes using 'Qwen3-8B-Instruct' while Section IV opens with 'While Qwen3-32B-Instruct provides strong zero-shot capabilities.' It is unclear whether 8B or 32B was actually used for fine-tuning."
     },
     {
       "flag": "Textbook math presented as contribution",
       "detail": "Sections III.A and IV present standard formulations (RoPE, GQA, SwiGLU, LoRA, PPO) as if they are contributions of the paper. These are established techniques reproduced from their original papers."
     },
     {
       "flag": "Ablation components may not be independent",
       "detail": "The ablation study removes one component at a time, but component improvements sum to 13.35pp while the full system improvement over direct generation is 10.67pp. This suggests component interactions or that the ablation baseline is not truly 'direct generation' as labeled."
     },
     {
       "flag": "Fine-tuning data undocumented",
       "detail": "The LoRA fine-tuning dataset is described only as 'historical GitHub issues with verified patches' with no information about size, source repositories, time period, or potential overlap with SWE-bench Lite."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. Narasimhan"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Defines the SWE-bench benchmark used to evaluate CodePilot; foundational benchmark for LLM-based program repair evaluation."
     },
     {
       "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
       "authors": ["H. Le", "Y. Wang", "A. D. Gotmare", "S. Savarese", "S. C. H. Hoi"],
       "year": 2022,
       "relevance": "Pioneered reinforcement learning for code generation, directly related to the PPO-based fine-tuning strategy used in CodePilot."
     },
     {
       "title": "Large Language Models Cannot Self-Correct Reasoning Yet",
       "authors": ["J. Huang", "X. Chen", "S. Mishra", "H. S. Zheng", "A. W. Yu", "X. Song", "D. Zhou"],
       "year": 2023,
       "arxiv_id": "2310.01798",
       "relevance": "Challenges the assumption that LLMs can self-correct, motivating CodePilot's use of external execution feedback rather than self-critique alone."
     },
     {
       "title": "StarCoder: May the Source Be with You!",
       "authors": ["R. Li", "L. B. Allal", "Y. Zi", "N. Muennighoff"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Major open-weight code LLM; relevant to the survey's coverage of code generation models and their capabilities."
     },
     {
       "title": "WizardCoder: Empowering Code Large Language Models with Evol-Instruct",
       "authors": ["Z. Luo", "C. Xu", "P. Zhao", "Q. Sun"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Instruction-tuning for code LLMs, directly relevant to understanding how fine-tuning affects code generation performance."
     },
     {
       "title": "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks",
       "authors": ["P. Lewis", "E. Perez", "A. Piktus", "F. Petroni", "V. Karpukhin"],
       "year": 2020,
       "relevance": "Foundational RAG paper; CodePilot's hybrid retrieval for fault localization builds on this approach."
     },
     {
       "title": "GraphCodeBERT: Pre-training Code Representations with Data Flow",
       "authors": ["D. Guo", "S. Ren", "S. Lu", "Z. Feng"],
       "year": 2020,
       "arxiv_id": "2009.08366",
       "relevance": "Structural code representations using data flow; relevant to CodePilot's AST-based code analysis."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["S. Yao", "D. Yu", "J. Zhao", "I. Shafran", "T. Griffiths", "Y. Cao", "K. Narasimhan"],
       "year": 2023,
       "relevance": "Tree-search approach for LLM reasoning; directly related to CodePilot's MCTS-guided generation strategy."
     },
     {
       "title": "Training Language Models to Follow Instructions with Human Feedback",
       "authors": ["L. Ouyang", "J. Wu", "X. Jiang", "D. Almeida", "C. Wainwright", "P. Mishkin"],
       "year": 2022,
       "relevance": "InstructGPT/RLHF paper; foundational to the PPO-based reinforcement learning from execution feedback used in CodePilot."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["N. Shinn", "F. Cassano", "A. Gopinath", "K. Narasimhan", "S. Yao"],
       "year": 2023,
       "relevance": "Self-reflective agents that improve through verbal feedback; directly related to CodePilot's execution-driven self-refinement mechanism."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 1,
       "justification": "Describes a potentially useful program repair framework but releases no code, making it impractical for adoption."
     },
     "surprise_contrarian": {
       "score": 0,
       "justification": "Confirms the expected finding that MCTS improves LLM code generation; no conventional wisdom is challenged."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk concerns are raised by this work."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict angle; a straightforward benchmark evaluation paper."
     },
     "demo_ability": {
       "score": 0,
       "justification": "No code, demo, or installable tool is provided."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Uses Qwen3 (moderately recognized) but the author and institution are not well-known in this space."
     }
   }
 }