ai-research-survey

scan.json (19861B)

---

 {
   "paper": {
     "title": "Confidence-Driven Multi-Scale Model Selection for Cost-Efficient Inference",
     "authors": ["Bo-Wei Chen", "Chung-Chi Chen", "An-Zi Yen"],
     "year": 2026,
     "venue": "",
     "arxiv_id": "",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository URL provided: https://github.com/NYCU-NLP-Lab/ConfDrivenInference (footnote 1, Section 2)."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Uses publicly available benchmarks: MMLU, GPQA, and PopQA. No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B specifies 4 NVIDIA RTX A6000 GPUs with 48GB VRAM, specific HuggingFace model identifiers, and generation parameters (do_sample=False, top_p=0.9, temperature=1.0)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README or reproduction guide is described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "McNemar's test is used for key comparisons (Appendix E, Table 9), with p-values reported for multiple configurations."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Performance drops and cost reductions are reported with baseline context (e.g., '83.22% vs. 83.57%' with '36% reduction in cost'), providing magnitude of effects."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for sample sizes or power analysis. MMLU has ~14k questions but no discussion of whether this is sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B explicitly states 'All results are based on a single run per model configuration without repeated sampling.' No variance or standard deviation reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Individual model performance (3B, 8B, 70B alone) serves as baselines, and comparisons are made against using the largest model directly."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No comparison against other routing/cascade methods like RouteLLM (Ong et al., 2024), Eagle (Zhao et al., 2024), or Hybrid LLM (Ding et al., 2024), which are discussed in related work but not compared experimentally."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 presents ablation of P(IK) component (Table 5), and Appendix I provides sensitivity analysis of P(T) threshold (Table 15)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Reports accuracy, Macro-Precision, Macro-Recall, Macro-F1 (Table 2), computational cost reduction, end-to-end time, and performance drop."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper measuring automated metrics on multiple-choice and QA tasks; human evaluation is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "For P(IK) classifier: '80% of the dataset, with 10% used for validation and 10% for testing' (Section 2.2). GPQA serves as an OOD test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "MMLU has 57 subjects but no per-subject breakdown is provided. Only aggregate results are reported."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 discusses failure on OOD (GPQA) where cost savings are minimal. Section 3.2 discusses error accumulation in longer chains. Section 4.1 discusses performance degradation without P(IK)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Reports that longer transfer chains (3B->8B->70B) lead to 'more pronounced performance drop' and that OOD generalization to GPQA yields only ~4-5% cost savings vs 36% on MMLU."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 20-40% cost reduction and ~60% token reduction for GPT-4o are supported by Tables 1 and 4 respectively."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are primarily from ablation studies (removing P(IK)) which use controlled single-variable manipulation (Table 5). The paper frames improvements as resulting from the confidence-driven strategy and supports this with ablations."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Abstract claims 'confidence-based model selection to enhance real-world LLM deployment' but experiments are limited to MMLU (multiple-choice NLU), GPQA, and PopQA. The title suggests broad applicability not supported by the narrow evaluation scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for results. For example, cost savings could be partly due to MMLU's question difficulty distribution rather than the confidence mechanism itself. No threats-to-validity section addresses this."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B lists exact HuggingFace model identifiers: meta-llama/Llama-3.2-3B-Instruct, meta-llama/Meta-Llama-3.1-8B-Instruct, meta-llama/Llama-3.3-70B-Instruct, Qwen/Qwen3-4B, Qwen/Qwen3-8B, Qwen/Qwen3-32B."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Table 7 (Appendix C) provides the full prompt templates for both multiple-choice and open-ended questions, with the fill values being the dataset's subject, question, and choices."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports do_sample=False, top_p=0.9, temperature=1.0. P(T) threshold set to 0.9 (Section 2.2). P(IK) classifier uses 80/10/10 split with hidden states from 24th transformer layer."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a cascade/routing mechanism, not an agent."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how MMLU, GPQA, or PopQA data was preprocessed. The train/val/test split for P(IK) is mentioned but selection criteria and any filtering are not documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section follows the Conclusions, discussing domain generalization, task coverage, and latency concerns."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Limitations section discusses specific issues: P(IK) classifier may degrade in different domains, experiments primarily focus on NLU tasks (MMLU/GPQA), and multi-model escalation may introduce latency."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Limitations explicitly state that applicability to 'generative language modeling, open-domain QA, or real-time conversational AI, remains an open challenge' and that results are primarily on NLU tasks."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (per-query predictions, confidence scores, routing decisions) is released. Only aggregate results in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Uses established public benchmarks (MMLU, GPQA, PopQA) with citations. P(IK) training data comes from model outputs on these benchmarks with 80/10/10 split."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark data to final results is not fully documented. For example, how P(IK) training labels were generated, how queries were formatted, and how PopQA grounding API verification worked are insufficiently described."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NSTC Taiwan grant, CITI Academia Sinica project, and AIST policy-based budget project."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: NYCU Taiwan and AIST Japan. No evaluated products are from these institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are government agencies (NSTC Taiwan, AIST Japan) and academic institutions (Academia Sinica) with no financial stake in the results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (LLaMA, Qwen, GPT-4o). MMLU was published in 2020 and is likely in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MMLU or GPQA questions appeared in model training data, despite MMLU being a widely-used benchmark likely contaminated in newer models."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU (2020) predates all models used. No contamination analysis is provided despite this being a known concern for MMLU evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix G (Tables 12-13) provides detailed token pricing and estimated USD costs for all configurations. GFLOPs computation costs are reported in Section 3.1 and Table 1."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware specified (4x NVIDIA RTX A6000). End-to-end time reported in Table 1. Total costs in USD estimated in Table 13. GFLOPs calculated for each configuration."
       }
     }
   },
   "claims": [
     {
       "claim": "The confidence-driven strategy achieves accuracy comparable to the largest model while reducing computational costs by 20% to 40%.",
       "evidence": "Table 1: 8B->70B achieves 83.22% vs 70B's 83.57% with 36.46% cost reduction. McNemar's test p=0.4048 shows no significant difference (Appendix E).",
       "supported": "strong"
     },
     {
       "claim": "When applied to GPT-4o, the method reduces token usage by approximately 60%.",
       "evidence": "Table 4: 70B->GPT-4o reduces tokens from 36,225 to 14,505 (59.96% reduction) while slightly improving accuracy (86.85% vs 86.43%).",
       "supported": "strong"
     },
     {
       "claim": "P(IK) classifier improves performance stability in the routing framework.",
       "evidence": "Table 5: With P(IK), 8B->70B drops only 0.35%; without P(IK), it drops 1.26%. Sensitivity analysis (Table 15) shows P(IK) stabilizes performance across threshold values.",
       "supported": "strong"
     },
     {
       "claim": "The method remains effective in out-of-distribution settings (GPQA).",
       "evidence": "Table 6: On GPQA, 8B->70B achieves 51.79% vs 52.23% with only 3.93% cost reduction. The cost savings are minimal compared to in-distribution MMLU.",
       "supported": "weak"
     },
     {
       "claim": "The method generalizes across model families (LLaMA and Qwen).",
       "evidence": "Table 3: Qwen 8B->32B achieves 80.00% vs 32B's 79.51% with 33.18% cost reduction, demonstrating cross-family effectiveness.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes a confidence-driven multi-scale model selection strategy combining token probability P(T) and a trained P(IK) classifier to route queries from smaller to larger LLMs. On MMLU, the 8B->70B cascade matches the 70B model's accuracy (83.22% vs 83.57%, p=0.4048) while reducing compute by 36%. When cascading to GPT-4o API, the 70B->GPT-4o configuration reduces token usage by ~60% with no accuracy loss. The approach generalizes to Qwen models but shows limited cost savings on the harder OOD benchmark GPQA (~4% vs ~36% on MMLU).",
   "red_flags": [
     {
       "flag": "No comparison with competing routing methods",
       "detail": "RouteLLM, Eagle, Hybrid LLM, and SYMBOLIC-MOE are discussed in related work but none are compared against experimentally, making it impossible to assess whether this method improves over existing approaches."
     },
     {
       "flag": "Single-run results",
       "detail": "All results are explicitly from a single run per configuration (Appendix B). No variance or standard deviation is reported, making result stability unknown."
     },
     {
       "flag": "MMLU contamination risk unaddressed",
       "detail": "MMLU was published in 2020 and is widely known to be contaminated in modern LLMs. No discussion of how contamination might affect the routing strategy's apparent effectiveness."
     },
     {
       "flag": "Weak OOD claim",
       "detail": "The abstract and conclusion claim effectiveness in 'both in-distribution and out-of-distribution settings,' but GPQA results show only 3.93-5.11% cost savings (vs 36-44% on MMLU), making the OOD claim overstated."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational work on LLM cascading with trained scoring functions for cost reduction."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2024,
       "relevance": "Competing routing approach using preference-based scoring for multi-LLM inference."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "relevance": "Competing approach training a query router to predict response quality gaps between small and large models."
     },
     {
       "title": "Eagle: Efficient Training-Free Router for Multi-LLM Inference",
       "authors": ["Zesen Zhao", "Shuowei Jin", "Z Morley Mao"],
       "year": 2024,
       "arxiv_id": "2409.15518",
       "relevance": "Training-free routing method for multi-LLM inference based on benchmark performance."
     },
     {
       "title": "Language Models (Mostly) Know What They Know",
       "authors": ["Saurav Kadavath"],
       "year": 2022,
       "arxiv_id": "2207.05221",
       "relevance": "Foundational work on P(True) and P(I Know) confidence calibration that this paper builds upon."
     },
     {
       "title": "Quantifying Uncertainty in Answers from Any Language Model and Enhancing Their Trustworthiness",
       "authors": ["Jiuhai Chen", "Jonas Mueller"],
       "year": 2024,
       "relevance": "LLM uncertainty quantification methods relevant to confidence-driven routing."
     },
     {
       "title": "Semantic Uncertainty: Linguistic Invariances for Uncertainty Estimation in Natural Language Generation",
       "authors": ["Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar"],
       "year": 2023,
       "arxiv_id": "2302.09664",
       "relevance": "Semantic entropy approach to LLM uncertainty estimation."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "authors": ["Jared Kaplan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Scaling laws used to estimate computational cost of model forward passes in this paper."
     },
     {
       "title": "Confident or Seek Stronger: Exploring Uncertainty-Based On-Device LLM Routing from Benchmarking to Generalization",
       "authors": ["Yu-Neng Chuang"],
       "year": 2025,
       "arxiv_id": "2502.04428",
       "relevance": "Concurrent work on uncertainty-based LLM routing for on-device deployment."
     },
     {
       "title": "Symbolic Mixture-of-Experts: Adaptive Skill-Based Routing for Scalable Heterogeneous Reasoning",
       "authors": ["Justin Chih-Yao Chen"],
       "year": 2025,
       "arxiv_id": "2503.05641",
       "relevance": "Symbolic skill-based routing approach for multi-LLM inference without gradient-based optimization."
     }
   ]
 }