ai-research-survey

scan.json (19412B)

---

 {
   "paper": {
     "title": "Confidence-Guided Stepwise Model Routing for Cost-Efficient Reasoning",
     "authors": ["Sangmook Lee", "Dohyung Kim", "Hyukhun Koh", "Nakyeong Yang", "Kyomin Jung"],
     "year": 2025,
     "venue": "AAAI 2026",
     "arxiv_id": "2511.06190"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive link found in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used (MATH500, OmniMath, AIME, MuSiQue, ACPBench, KOR-Bench) are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using vLLM but does not provide a requirements.txt, Dockerfile, or detailed environment specification with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts are provided. Algorithm 1 describes the method but not how to run experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1 and 2 report only point estimates for accuracy and FLOPs with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims STEER outperforms baselines but provides no statistical significance tests. Comparisons are based solely on comparing numbers."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context, e.g., '+20% accuracy with 48% less FLOPs compared to solely using the larger model on AIME', and provides full baseline and method numbers in tables."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes or number of runs. The paper does not discuss whether the sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against RSD, Damani et al. (2025), and SpecReason, plus small-only and large-only baselines (Tables 1 and 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2024-2025 (RSD, SpecReason, Damani et al. 2025), representing recent work in the cost-efficient reasoning space."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 includes ablations on confidence measure selection (Figure 3), group size variations (Table 3), and larger model usage pattern analysis (Figure 5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Accuracy, FLOPs, and Accuracy-per-FLOPs (A/F) across all experiments."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates automated reasoning benchmarks with ground-truth answers; human evaluation is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "All benchmarks used are standard test sets (MATH500, AIME, OmniMath, MuSiQue, ACPBench, KOR-Bench). The GMM is fit on the same benchmark questions at inference time, but no training/tuning is done on these sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (Tables 1, 2), by difficulty level (Figure 5 on OmniMath), and by step position."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses where STEER has lower accuracy than RSD on Qwen2.5-Math-Instruct (Section 4.2), and analyzes larger model usage on incorrect solutions (Figure 5)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports cases where STEER does not outperform RSD (e.g., Qwen2.5-Math-Instruct on math benchmarks) and where accuracy drops relative to the large model only."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims '+20% accuracy with 48% less FLOPs on AIME' which is supported by Table 1 (Gemma3 AIME: 15.8 vs 17.5 accuracy, 30.6 vs 33.8 FLOPs — though the 20% figure seems to be relative to a different comparison point). The claim of outperforming trained external module baselines is supported in most but not all settings."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims through ablation studies (confidence measure selection in Figure 3, group size in Table 3) which use controlled single-variable manipulation. The claim that confidence-based routing causes cost reduction is supported by the framework design."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract claim 'domain-agnostic' routing, but results are on 6 benchmarks across only math, QA, and planning. The paper does not bound claims to these specific domains and generalizes to 'cost-efficient reasoning' broadly."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why STEER works, such as whether the performance gains could be attributed to the specific threshold tuning rather than the confidence signal itself."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'Gemma3-Instruct 4B/12B', 'Qwen2.5-Instruct 1.5B/7B', and 'Qwen2.5-Math-Instruct 1.5B/7B' without specifying exact version snapshots or release dates."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes using prompts for reasoning but does not provide the actual prompt text used in experiments. The appendix references (A, B) are not included in the available text."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature=0.7 is specified. The threshold γ is selected via grid search with gap 0.1. The GMM fitting procedure is described. vLLM is specified as the inference engine."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The step-level routing mechanism is described in detail in Section 3, with formal algorithm (Algorithm 1), confidence estimation (Section 3.1), GMM-based routing (Section 3.2), and the full STEER procedure (Section 3.3)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents benchmark selection, exclusion of Operation and Puzzles subsets from KOR-Bench with justification, and exclusion of model families from certain benchmarks with reasons stated."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section found in the paper."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or which settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-question results, confidence scores, routing decisions) is made available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The benchmarks are standard and well-referenced. AIME questions are described as 'collected from 2022 through 2025'. Benchmark details are in Section 4.1."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; all data comes from standard benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper describes the full pipeline: benchmark selection, model selection with exclusion criteria, threshold grid search procedure, and FLOPs computation method."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Acknowledgments) lists IITP grants funded by the Korea government (MSIT) and thanks LYWAY for experimental support."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from Seoul National University, clearly stated in the paper header."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding is from Korean government research grants (IITP/MSIT) which have no financial stake in the specific routing method's performance. LYWAY provided API support but the paper does not evaluate LYWAY products."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for the models used (Gemma3, Qwen2.5)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MATH500 or other benchmarks may have been in the training data of the models evaluated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MATH500 (2021) and other benchmarks predate the models used, raising contamination risk that is not addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "FLOPs per query are reported for all methods across all benchmarks (Tables 1, 2). Routing latency comparison is in Figure 4."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No mention of total GPU hours, hardware used, or total computational budget for the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "STEER achieves +20% accuracy with 48% less FLOPs compared to solely using the larger model on AIME",
       "evidence": "Table 1 shows Gemma3-Instruct STEER on AIME: 15.8% accuracy, 30.6 FLOPs vs 12B only: 17.5% accuracy, 33.8 FLOPs. The numbers don't directly show +20% accuracy; the claim likely refers to comparison with 4B only (10.8% accuracy).",
       "supported": "weak"
     },
     {
       "claim": "STEER outperforms baselines that rely on trained external modules across diverse benchmarks",
       "evidence": "Tables 1 and 2 show STEER achieves highest average A/F in most settings. However, RSD outperforms STEER on Qwen2.5-Math-Instruct math benchmarks in accuracy.",
       "supported": "moderate"
     },
     {
       "claim": "STEER is robust to changes in backbone LLM family",
       "evidence": "Experiments on both Gemma3-Instruct and Qwen2.5 families (Tables 1, 2) show STEER works across model families, unlike RSD which degrades on Gemma3.",
       "supported": "moderate"
     },
     {
       "claim": "GMM routing latency is over a magnitude lower than baseline approaches",
       "evidence": "Figure 4 shows routing latency comparison on MATH500 benchmark.",
       "supported": "moderate"
     },
     {
       "claim": "Maximum logit is the most effective confidence signal for routing",
       "evidence": "Figure 3 compares logit, entropy, and maximum probability on MATH500, showing logit yields highest accuracy especially at higher routing percentiles.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "STEER proposes using logit-based confidence scores with Gaussian Mixture Models for step-level routing between small and large LLMs during reasoning, without requiring trained external models. Across 6 benchmarks spanning math, QA, and planning tasks, STEER achieves competitive accuracy while reducing inference FLOPs by 10-48% compared to using only the larger model. The method shows greater cross-model robustness than PRM-based baselines like RSD, which degrade when applied to model families different from their training data.",
   "red_flags": [
     {
       "flag": "No variance or error bars",
       "detail": "All results appear to be single-run numbers with no standard deviation, confidence intervals, or multiple-seed averaging. Given the stochastic nature of LLM generation with temperature=0.7, results could vary substantially across runs."
     },
     {
       "flag": "Questionable abstract claim",
       "detail": "The abstract claims '+20% accuracy with 48% less FLOPs on AIME' but Table 1 shows STEER at 15.8% vs 12B-only at 17.5% accuracy on AIME with Gemma3. The +20% claim is unclear in its reference point and may be cherry-picked."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no dedicated limitations or threats-to-validity discussion, which is a notable omission for an empirical methods paper."
     },
     {
       "flag": "Benchmark contamination risk unaddressed",
       "detail": "MATH500 (2021) and other benchmarks predate the models used. No discussion of whether contamination affects the absolute accuracy numbers, though the relative comparisons between methods may be less affected."
     },
     {
       "flag": "Threshold tuning on test set",
       "detail": "Grid search over threshold values is performed selecting the 'best-performing value' — it is unclear if this is done on the test set itself, which would inflate results for all methods including STEER."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "relevance": "Foundational work on cost-efficient LLM inference through model cascading and routing."
     },
     {
       "title": "Reward-Guided Speculative Decoding for Efficient LLM Reasoning",
       "authors": ["Baohao Liao"],
       "year": 2025,
       "relevance": "Key baseline (RSD) for step-level model routing using process reward models."
     },
     {
       "title": "Learning How Hard to Think: Input-Adaptive Allocation of LM Computation",
       "authors": ["Mehul Damani"],
       "year": 2025,
       "relevance": "Key baseline for query-level model allocation using trained difficulty estimators."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong"],
       "year": 2024,
       "relevance": "Trained router for LLM model selection, relevant to cost-efficient LLM deployment."
     },
     {
       "title": "Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws",
       "authors": ["Nikhil Sardana"],
       "year": 2024,
       "relevance": "Scaling laws accounting for inference costs, foundational to the cost-efficiency motivation."
     },
     {
       "title": "Stop overthinking: A survey on efficient reasoning for large language models",
       "authors": ["Yuan Sui"],
       "year": 2025,
       "relevance": "Survey of efficient reasoning methods for LLMs, directly relevant to the survey scope."
     },
     {
       "title": "CoT-Valve: Length-Compressible Chain-of-Thought Tuning",
       "authors": ["Xinyin Ma"],
       "year": 2025,
       "relevance": "Method for compressing chain-of-thought reasoning to reduce inference cost."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding"],
       "year": 2024,
       "relevance": "Query routing between different-sized LLMs for cost efficiency."
     },
     {
       "title": "Detecting hallucinations in large language models using semantic entropy",
       "authors": ["Sebastian Farquhar"],
       "year": 2024,
       "relevance": "Semantic entropy for LLM uncertainty quantification, relevant to confidence estimation approaches."
     },
     {
       "title": "Deepseek-r1: Incentivizing reasoning capability in llms via reinforcement learning",
       "authors": ["Daya Guo"],
       "year": 2025,
       "relevance": "Major reasoning LLM using RL-based test-time computation, motivating the need for cost-efficient reasoning."
     }
   ]
 }