ai-research-survey

scan.json (26780B)

---

 {
   "paper": {
     "title": "Dynamic Mix Precision Routing for Efficient Multi-step LLM Interaction",
     "authors": [
       "Yuanzhe Li",
       "Jianing Deng",
       "Jingtong Hu",
       "Tianlong Chen",
       "Song Wang",
       "Huanrui Yang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.02711"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Sensitivity to model quantization in multi-step agentic tasks is concentrated at a small number of critical decision steps. A lightweight 2-layer Transformer router trained via KL-divergence supervision and GRPO can dynamically select between high- and low-precision LLMs at each step, achieving near full-precision performance (88.8% vs 89.6% on ALFWorld for Qwen3-8B) while using only ~27% high-precision calls. GRPO improves routing efficiency on larger models but provides no gains on the smallest model (Qwen3-1.7B), suggesting routing effectiveness is bounded by the high-precision model's capability.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses ALFWorld, a publicly available benchmark environment (Shridhar et al., 2021). No proprietary data was collected."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper specifies model names and quantization methods (GPTQ 3-bit/4-bit) but provides no requirements.txt, Dockerfile, or detailed environment setup with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or scripts for replicating experiments are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 and Table 2 report only point estimates for success rate, high-precision ratio, and GHC. No confidence intervals, error bars, or ± notation appear anywhere."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims routing 'achieves a great improvement' and 'consistently achieves superior performance–cost trade-offs' but provides no statistical significance tests — all comparisons are based on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 provides success rates for all methods alongside their high-precision ratios, and the GHC metric quantifies improvement magnitude per unit of high-precision cost. For example, the router achieves 88.8% (vs 82.8% quantized-only baseline) with 26.7% HP ratio, giving sufficient context to understand effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the number of evaluation episodes used. The number of ALFWorld test tasks is not stated explicitly, nor is any power analysis provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or spread measures are reported. All results appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against full-precision BF16, quantized-only (GPTQ), and random routing at 20/40/60/80% high-precision ratios across four model configurations."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "The baselines are only random routing at various ratios and single-precision inference. No comparison with existing routing methods from the related work (FrugalGPT, RouteLLM, HybridLLM, BEST-Route, Router-R1) despite extensive discussion of them in Section 2."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 ablates KL-ST vs GRPO-only vs KL-ST+GRPO. Table 3 ablates KL-ST training data scale (100/200/300/400 episodes). Both are systematic single-variable manipulations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are reported: success rate, high-precision usage ratio, and Gain per High-Precision Call (GHC). These capture effectiveness, cost, and their trade-off."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates routing in an automated simulator (ALFWorld) with binary task success. Human evaluation is not relevant to the claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 states 'we evaluate on the unseen test task following Yao et al. (2022b),' using ALFWorld's standard unseen test split."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld has 6 task types (pick, clean, heat, cool, examine, pick-two) but no per-task-type breakdown is provided. Results are only broken down by model configuration, not by task category."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 and Appendix A.2 provide detailed case studies of trajectories where the low-precision model fails at critical steps while the router succeeds, with step-by-step traces."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 shows GRPO provides no improvement on Qwen3-1.7B (identical results to KL-ST). Section 5.3 discusses this: 'the effectiveness of routing-based optimization is bounded by the expressiveness and capability of the high-precision model itself.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'great improvement on accuracy–cost trade-off over single-precision baselines and heuristic routing methods.' Table 1 supports this: the router achieves the highest GHC across all model configurations."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The ablation study (Table 2) uses controlled single-variable manipulation: KL-ST only, GRPO only, and KL-ST+GRPO. This is an adequate design for the causal claims about component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Multi-step LLM Interaction' generally, and Section 1 claims the framework is for 'long-horizon agentic tasks.' But experiments are only on ALFWorld — no evaluation on other agentic benchmarks (WebArena, ScienceWorld, SWE-bench) despite citing them in related work."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, whether the router merely learns to predict task difficulty rather than precision sensitivity, or whether simple heuristics (e.g., step position in trajectory) could explain the routing decisions."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures what it claims: task success rate and high-precision usage ratio. The GHC metric is explicitly defined as combining these two. No proxy gap exists between measurement and claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are specified as 'Qwen3-8B', 'Qwen3-4B', 'Qwen3-1.7B', and 'DeepSeek-R1-Distill-Llama3-8B' without specific checkpoint versions, HuggingFace model IDs, or snapshot dates. 'Post-training form released by the authors' is vague."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses ReAct-style prompting for ALFWorld but does not provide the actual prompt text. Only interaction traces (action/observation pairs) are shown in examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B reports training hyperparameters: learning rate 1e-4, batch size 64, 5 epochs for KL-ST; learning rate 1e-6, βKL=0.02, STRONG_COST=0.02 for GRPO. Router threshold between 78th-85th percentile of KL distribution."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The routing framework is described in detail: Section 4.1 covers the 2-layer Transformer encoder architecture, step-level state representation, positional embeddings, and masked pooling. Section 3 formalizes the routing decision process. Figures 2 and 3 show the architecture."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the trajectory sampling protocol: high-precision rollouts, retention of successful trajectories only, step-wise KL computation, CDF-based thresholding to binary labels, and class-specific weighting for imbalance."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations, threats to validity, or discussion section exists. The conclusion (Section 6) is 4 sentences with no discussion of limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper does not acknowledge limitations of testing on only one benchmark or one task domain."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data (trajectories, KL divergence values, routing decisions) is released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the trajectory sampling protocol in detail: 200 episodes from high-precision rollouts, filtering for successful trajectories, computing action distributions from both precision levels at each step."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data source is ALFWorld, a standard public benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: high-precision rollouts → filter successful trajectories → compute step-wise KL → apply CDF threshold for binary labels → class-weighted training. GRPO pipeline (120 episodes, K rollouts per instance) is also described in Appendix B."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Arizona, University of Pittsburgh, University of North Carolina at Chapel Hill, University of Central Florida. All academic institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The paper is a multi-university collaboration where funding would typically exist."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for Qwen3 or DeepSeek-R1-Distill models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' training data could include ALFWorld task descriptions, solutions, or related content."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "ALFWorld was published in 2021. All models used (Qwen3, DeepSeek-R1-Distill) were trained after 2021. No contamination analysis is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "High-precision usage ratio is reported for all methods (Table 1), which directly measures the fraction of expensive inference calls. The GHC metric explicitly captures cost-effectiveness. However, no actual wall-clock time or dollar costs are provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training uses 200 episodes for KL-ST and 120 for GRPO (Appendix B), but no GPU hours, wall-clock time, or hardware specifications are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results appear to be single-seed runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation runs per configuration is never stated. Results are presented as point estimates without clarifying how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The KL threshold τ is 'manually selected based on the empirical distribution' (Appendix B) between 78th-85th percentiles. No systematic search budget is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B states: 'Model selection is performed by choosing the checkpoint that achieves the highest validation accuracy across all training epochs.' Selection criterion and split are specified."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction does not apply."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The baselines (random routing) are all implemented by the authors. No discussion of self-evaluation bias or comparison with independently implemented baselines."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 and Table 1 plot success rate against high-precision usage ratio, which directly measures compute allocation. The GHC metric normalizes performance gain by compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.1 asserts ALFWorld is 'a representative benchmark for evaluating agentic ability' but does not critically assess whether ALFWorld's text-based household tasks actually measure the 'long-horizon decision making' capability claimed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "All comparisons use the same ReAct-based interaction framework with ALFWorld. The only variable is precision routing — the scaffold is held constant across all conditions."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the models' training data included ALFWorld-related content or solutions from after the benchmark's release."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information through context that would not be available in genuine deployment."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training episodes (used for router training) and test episodes share structural similarities or task templates."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied."
       }
     }
   },
   "claims": [
     {
       "claim": "The router achieves near full-precision performance with less than 30% high-precision calls on Qwen3-8B",
       "evidence": "Table 1: Router achieves 88.8% success rate with 26.7% high-precision ratio vs BF16's 89.6% at 100%. GHC of 19.85 vs best random routing GHC of 8.5.",
       "supported": "strong"
     },
     {
       "claim": "Sensitivity to quantization is concentrated at a small number of critical decision steps",
       "evidence": "Figure 4 shows the step-wise KL divergence distribution between 3-bit and bf16 Qwen-8B is highly skewed, with most steps near zero and a heavy tail of high-divergence steps.",
       "supported": "strong"
     },
     {
       "claim": "GRPO further improves routing efficiency over KL-ST alone",
       "evidence": "Table 2: On Qwen3-4B, GHC improves from 26.43 (KL-ST) to 43.02 (KL-ST+GRPO). On Qwen3-8B, from 18.79 to 19.85. However, no improvement on Qwen3-1.7B (both 21.95) or marginal on DeepSeek (30.61 to 38.77).",
       "supported": "moderate"
     },
     {
       "claim": "A lightweight 2-layer Transformer encoder is sufficient for step-level precision routing",
       "evidence": "Section 4.1 and results in Table 1 demonstrate the architecture works. However, no comparison with larger routers, simpler routers, or other architectures is provided to establish that 2 layers is the right size.",
       "supported": "weak"
     },
     {
       "claim": "The effectiveness of routing is bounded by the high-precision model's capability",
       "evidence": "Table 2: GRPO provides largest GHC improvement on Qwen3-4B (highest base success rate at 93.3% BF16) and zero improvement on Qwen3-1.7B (lowest at 69.4% BF16). Section 5.3 discusses this pattern.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Single benchmark evaluation",
       "detail": "All experiments are on ALFWorld only, despite the paper citing multiple agentic benchmarks (WebArena, ScienceWorld, SWE-bench) in related work. Claims about 'long-horizon agentic tasks' generally are not supported by a single text-based household environment."
     },
     {
       "flag": "No error bars or variance reporting",
       "detail": "All results in Tables 1-3 are point estimates with no standard deviations, confidence intervals, or indication of how many runs produced each number. Given that ALFWorld episodes involve stochastic sampling, results could vary significantly across runs."
     },
     {
       "flag": "Missing comparison with existing routing methods",
       "detail": "The related work section discusses FrugalGPT, RouteLLM, HybridLLM, BEST-Route, and Router-R1 extensively, but none are included as baselines. Only random routing (a trivially weak baseline) is compared."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats to validity, or future work discussion. The 4-sentence conclusion acknowledges no weaknesses."
     },
     {
       "flag": "GHC metric can mask absolute performance gaps",
       "detail": "For Qwen3-4B, the router achieves 81.3% success (with GHC 43.02) while Random@60% achieves 88.8% success (GHC 18.67). The high GHC rewards extreme cost reduction but the router's success rate is 7.5 percentage points lower than an achievable configuration."
     }
   ],
   "cited_papers": [
     {
       "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
       "authors": ["Shunyu Yao", "Jeffrey Zhao", "Dian Yu", "Nan Du", "Izhak Shafran", "Karthik R. Narasimhan", "Yuan Cao"],
       "year": 2022,
       "relevance": "Foundational LLM reasoning-and-acting framework used as the agent backbone in this study."
     },
     {
       "title": "Toolformer: Language Models Can Teach Themselves to Use Tools",
       "authors": ["Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì"],
       "year": 2023,
       "relevance": "Key work on LLMs learning to use external tools, relevant to agentic AI capability evaluation."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLM agents on real-world software engineering tasks."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "relevance": "LLM agent system for software engineering, representative of agentic AI capabilities."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2024,
       "relevance": "Pioneering cascading framework for cost-efficient LLM inference, directly comparable to routing approaches."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2025,
       "arxiv_id": "2406.18665",
       "relevance": "Learned routing policies between LLMs using preference data, a direct precursor to this work."
     },
     {
       "title": "Can Compressed LLMs Truly Act? An Empirical Evaluation of Agentic Capabilities in LLM Compression",
       "authors": ["Peiyan Dong", "Zhangchi Tang", "Xinyu Liu"],
       "year": 2025,
       "relevance": "Empirical study showing significant performance degradation when quantized LLMs are used in agentic settings — the core motivation for this paper."
     },
     {
       "title": "Quantization Meets Reasoning: Exploring LLM Low-Bit Quantization Degradation for Mathematical Reasoning",
       "authors": ["Zhen Li", "Yupeng Su", "Runming Yang"],
       "year": 2025,
       "arxiv_id": "2501.03035",
       "relevance": "Studies quantization degradation in LLM reasoning, complementary evidence for precision sensitivity."
     },
     {
       "title": "BEST-Route: Adaptive LLM Routing with Test-Time Optimal Compute",
       "authors": ["Dujian Ding", "Ankur Mallick", "Shuai Zhang"],
       "year": 2025,
       "relevance": "LLM routing with test-time compute allocation, directly comparable routing approach."
     },
     {
       "title": "Router-R1: Teaching LLMs Multi-Round Routing and Aggregation via Reinforcement Learning",
       "authors": ["Hao Zhang", "Tao Feng", "Jieyu You"],
       "year": 2025,
       "arxiv_id": "2506.09033",
       "relevance": "Formulates routing as sequential decision process with RL, the closest prior work to this paper's approach."
     },
     {
       "title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
       "authors": ["Guanzhi Wang", "Yuqi Xie", "Yunfan Jiang"],
       "year": 2024,
       "relevance": "Open-ended LLM agent for embodied tasks, representative of the agentic AI paradigm this paper targets."
     },
     {
       "title": "Efficient Agents: Building Effective Agents While Reducing Cost",
       "authors": ["Nan Wang", "Xuemei Hu", "Pengfei Liu"],
       "year": 2025,
       "arxiv_id": "2508.02694",
       "relevance": "Directly addresses cost reduction for LLM agents, the same problem space as this paper."
     },
     {
       "title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "LLM agent self-improvement through verbal feedback, relevant to agentic AI methodology."
     }
   ]
 }