ai-research-survey

scan-v5.json (28377B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Dynamic Mix Precision Routing for Efficient Multi-step LLM Interaction",
     "authors": [
       "Yuanzhe Li",
       "Jianing Deng",
       "Jingtong Hu",
       "Tianlong Chen",
       "Song Wang"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.02711",
     "doi": null
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All four abstract claims are supported: (1) LLMs succeed at long-horizon tasks (shown in intro), (2) inference cost is prohibitive (referenced in related work), (3) the framework is described in Section 4, (4) Table 1 demonstrates accuracy-cost improvements over baselines.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation study (Section 5.3, Table 2) isolates KL-ST vs GRPO effects; comparisons against random routing and full-precision baselines support causal claims. However, limited to one benchmark environment.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "Paper frames problem broadly ('long-horizon agentic tasks', 'real-world scenarios') but evaluates only on ALFWorld. No explicit scope boundaries stated. Claims like 'effective precision routing does not require a full-capacity model' generalize beyond tested domain without justification.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations. For instance, GRPO shows no improvement on 1.7B model (Table 2) but paper only attributes this to 'inherent limitation' without exploring other factors. No consideration of confounds.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Measured outcomes (success rate, high-precision usage, GHC metric) map directly to intended claims (task performance and inference efficiency). No conflation between proxy and ground truth.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section. Conclusion (Section 6) is one paragraph summarizing findings without acknowledging scope boundaries.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats discussed. Single-environment evaluation, limited quantization methods (GPTQ only), ad-hoc hyperparameter selection (KL threshold 'manually selected based on empirical distribution'), and inconsistent GRPO gains across model scales are not acknowledged.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit boundaries stated. Paper does not acknowledge what the approach does NOT apply to (e.g., single-turn QA, retrieval tasks, or models without quantized variants).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement or acknowledgments section visible in paper. No disclosure of funding sources.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations listed at top (University of Arizona, University of Pittsburgh, UNC Chapel Hill, University of Central Florida). Affiliations are disclosed; however, no explicit conflict-of-interest statement provided.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "No funder identified.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing-interests or financial-disclosure statement included. Paper evaluates Qwen and DeepSeek models but does not disclose any potential financial relationships.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Core terms are defined: 'agentic tasks' as long-horizon decision-making requiring tool use and environment interaction (Section 1), 'dynamic mix-precision routing' explained with architecture (Section 4), 'quantization' used consistently with standard meaning.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three contributions explicitly stated in introduction: (1) dynamic framework for step-level routing, (2) RL optimization of routing policy, (3) lightweight router sufficiency. Clear and distinct contributions.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 engages with three areas: LLM routing (FrugalGPT, HybridLLM, RouteLLM, etc.), agentic tasks, and quantization. Paper positions this work as step-level routing (vs. query-level) on agentic tasks, showing how it addresses limitations of prior routing work.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": false,
           "justification": "No code repository, GitHub link, or supplementary code provided. Paper is a preprint with no mention of code availability.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "ALFWorld benchmark is publicly available and evaluation follows standard protocol (Yao et al. 2022b unseen test set). However, the 200 training rollouts used for KL-ST supervision are not released.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or environment specification provided. Training details in Appendix B (batch size, learning rate) but no computational environment or dependency specs.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "Experimental setup described in Section 5 and Appendix, but no step-by-step reproduction instructions. No code release makes reproduction difficult or impossible.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Table 1 reports success rates and metrics as point estimates with no confidence intervals, error bars, or standard deviations. Single evaluation run implied; no variance reporting.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests (t-tests, Mann-Whitney, etc.) reported. Comparisons presented descriptively (e.g., 88.8% vs 89.6%) without p-values or significance indicators.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "GHC metric directly measures effect size (gain per high-precision call). Improvements are quantified in percentage points. However, no formal effect-size statistics (Cohen's d, etc.) or confidence bounds reported.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "KL-ST uses 200 episodes (stated in Appendix B), GRPO uses 120 episodes. No power analysis or justification for these choices. Table 3 ablates data scale (100–400) showing variance in results but final choice not justified.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, std dev, or repeated runs reported. Especially problematic for RL results which are typically high-variance. Only point estimates in tables.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Comparisons include full-precision baseline (BF16), quantized-only baseline (GPTQ), and random routing at multiple thresholds (20%, 40%, 60%, 80%). Strong baseline set.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": false,
           "justification": "BF16 and GPTQ baselines are contemporary. However, no comparison to other learned routing methods mentioned in related work (RouteLLM, BEST-Route, RouterDC, etc.), only random and single-precision baselines.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Section 5.3 compares KL-ST-only, GRPO-only, and KL-ST+GRPO (Figure 5, Table 2). Table 3 ablates training data scale (100–400 episodes). Ablations show both components contribute.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Success rate (primary), high-precision usage ratio, and GHC (unified efficiency metric) reported. Multiple perspectives on performance–cost trade-off.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Not applicable. Task success is objective (environment-based); no human evaluation needed or provided.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Evaluation on 'unseen test task' following Yao et al. (2022b); ALFWorld has standard train/test split.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": false,
           "justification": "Results broken down by model family (Qwen, DeepSeek) and size (8B, 4B, 1.7B) but no breakdown by task type, difficulty, or failure mode within ALFWorld.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Appendix A.2 provides one qualitative case study (cleaning cloth task) showing low-precision failure and router success. No systematic failure analysis or taxonomy of failure modes.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "GRPO-only achieves lower GHC than KL-ST+GRPO (Table 2). GRPO provides no benefit on 1.7B model. Table 3 shows GHC fluctuations with data scale. Some negative/mixed results included.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": false,
           "justification": "Model families and sizes specified (Qwen3-8B/4B/1.7B, DeepSeek-R1-Distill-Llama3-8B) but no version snapshots or release dates. For reproducibility, commit hashes or exact model cards needed.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "No system prompt or instruction template provided. For ALFWorld this may be standard, but explicit prompt text is absent. Critical for reproducibility of LLM outputs.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "KL-ST hyperparameters provided (batch 64, lr 1e-4, 5 epochs, Appendix B). GRPO hyperparameters provided (lr 1e-6, KL weight 0.02, 120 episodes). However, KL threshold selection described as 'manually selected between 78th and 85th percentiles'—vague and ad-hoc.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "Router architecture detailed in Section 4 and Figure 3: 2-layer Transformer encoder with position embeddings, masked attention, and softmax classification. Scaffolding is the routing framework itself and is well-described.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": false,
           "justification": "Section 4.1 states step embeddings are 'pre-computed by an external encoder and treated as fixed inputs' but does not specify which encoder, how it works, or its parameters. Critical preprocessing detail missing.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "ALFWorld is publicly available. However, the 200 high-precision rollouts collected for KL-ST training are not released, limiting reproducibility of training data.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 describes trajectory sampling protocol: high-precision rollout strategy, measurement of step-wise KL divergence, retention of successful trajectories only. Well-documented collection process.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human subjects involved.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "Pipeline from trajectory collection → KL divergence computation → KL-ST training → GRPO optimization is documented in Sections 4.2–4.3 and Appendix B with sufficient detail.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "No training cutoff date stated for Qwen3 or DeepSeek-R1 models. These are 2024–2025 models but exact training data cutoff unknown. ALFWorld cutoff not discussed.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of potential overlap between ALFWorld test set and base model training data. With models released in 2025–2026 evaluating a 2021 benchmark, contamination risk is not addressed.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "ALFWorld (2021) may have been included in base model training data. Paper does not acknowledge or test for this risk.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "High-precision usage ratio and GHC metric quantify inference cost trade-offs. However, actual latency, wall-clock time, or GPU hours are NOT reported. Cost is relative, not absolute.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Episode budgets stated (200 for KL-ST, 120 for GRPO) but no compute cost in GPU hours, FLOPs, or dollars. Cannot assess practical compute requirements.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Low-precision quantized LLMs exhibit step-wise diversity in sensitivity: most steps tolerate quantization but a small fraction are 'critical' and require high precision.",
       "evidence": "Figure 4 shows step-wise KL divergence distribution is 'highly skewed' with heavy right tail across Qwen3-8B, Qwen3-4B, and DeepSeek models. Section 1 and 4.2 frame this observation.",
       "supported": "strong"
     },
     {
       "claim": "Dynamic step-level mix-precision routing achieves superior accuracy–cost trade-off compared to random routing and full-precision baselines.",
       "evidence": "Table 1: Router achieves GHC scores (19.85–43.02) consistently higher than random routing (2–18.67) across all models. Approaches full-precision performance (89.6% vs 88.8% on Qwen3-8B) with 26.7% high-precision usage.",
       "supported": "strong"
     },
     {
       "claim": "KL-divergence-based supervised learning (KL-ST) identifies precision-sensitive steps better than unsupervised methods.",
       "evidence": "Figure 5 and Table 2: KL-ST achieves higher GHC than GRPO-only (18.79 vs 10.74 on Qwen3-8B). KL-ST provides 'stable initialization' for downstream RL. Table 3 shows KL-ST data scale correlates with GHC improvement.",
       "supported": "moderate"
     },
     {
       "claim": "Group-Relative Policy Optimization (GRPO) further improves routing decisions over KL-ST alone.",
       "evidence": "Figure 5 and Table 2: KL-ST+GRPO (GHC 19.85) outperforms KL-ST-only (18.79) on Qwen3-8B. However, on smaller models (1.7B), GRPO provides NO improvement. Gain correlates with base model capability, not uniform.",
       "supported": "moderate"
     },
     {
       "claim": "A lightweight two-layer Transformer router is sufficient for step-level precision routing without requiring a full-capacity LLM.",
       "evidence": "Section 4.1 and Figure 3 describe router architecture (2 layers, position embeddings, last-token pooling). No comparison to larger or smaller routers provided. No justification for architecture choice.",
       "supported": "weak"
     },
     {
       "claim": "The approach generalizes across LLM families and model scales (Qwen, DeepSeek; 1.7B–8B).",
       "evidence": "Table 1 shows results on Qwen3-8B/4B/1.7B and DeepSeek-R1-Distill-Llama3-8B. However, all evaluation is on ALFWorld only. No evaluation on other agentic benchmarks (WebArena, ScienceWorld, etc.).",
       "supported": "weak"
     },
     {
       "claim": "The router efficiently reduces high-precision calls without sacrificing task success (e.g., 26.7% high-precision usage achieves 88.8% success vs 100% for full-precision at 89.6%).",
       "evidence": "Table 1: Qwen3-8B router uses 26.7% high-precision and achieves 88.8% success vs 100% high-precision at 89.6% success. GHC=19.85 indicates efficient trade-off. Results limited to ALFWorld.",
       "supported": "strong"
     },
     {
       "claim": "Routing decisions based on behavioral divergence (KL) are more effective than random routing at equivalent cost budgets.",
       "evidence": "Table 1: Router@26.7% high-precision (GHC 19.85) outperforms Random@20% (GHC 2) and Random@40% (GHC 8.5) on Qwen3-8B. Similar pattern across all models.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "empirical"
   ],
   "key_findings": "The paper demonstrates that step-wise precision sensitivity in quantized LLMs is heterogeneous—most steps tolerate quantization but critical decision points require full precision. A lightweight two-layer Transformer router trained via KL-divergence supervision and policy optimization can dynamically select between high- and low-precision models, achieving 26.7–88% high-precision usage while maintaining near-full-precision task success (88.8% vs 89.6%) on ALFWorld. The approach consistently outperforms random routing baselines, though improvements diminish on smaller models (1.7B), and has been validated only on a single benchmark environment.",
   "red_flags": [
     {
       "flag": "Single-benchmark evaluation",
       "detail": "All experiments on ALFWorld only. No evaluation on WebArena, ScienceWorld, or other agentic benchmarks limits claim generalization to 'real-world agentic tasks' (Section 1, page 1)."
     },
     {
       "flag": "No confidence intervals or repeated runs",
       "detail": "Table 1 reports point estimates with no error bars, std dev, or repeated evaluations. Single-run results; RL typically exhibits high variance, especially with sparse rewards (problem stated in Section 4.3)."
     },
     {
       "flag": "Missing external encoder specification",
       "detail": "Step embeddings pre-computed by 'external encoder' (Section 4.1, page 3) but encoder type, architecture, and training unknown. Critical preprocessing detail omitted, hindering reproducibility."
     },
     {
       "flag": "Ad-hoc hyperparameter selection",
       "detail": "KL threshold selected 'manually between 78th and 85th percentiles' (Appendix B, page 13). No ablation on threshold value. Selection method not principled."
     },
     {
       "flag": "Inconsistent GRPO gains by model scale",
       "detail": "GRPO improves GHC on Qwen3-4B (26.43→43.02) and 8B (18.79→19.85) but provides zero improvement on 1.7B (Table 2). Attribution to 'inherent router limitation' lacks depth; alternative explanations unexplored."
     },
     {
       "flag": "No comparison to learned routing baselines",
       "detail": "Section 2 cites RouteLLM, BEST-Route, RouterDC, and others, but experiments compare only to random routing and single-precision baselines. No head-to-head comparison to other routers."
     },
     {
       "flag": "Training data scale variance unexplained",
       "detail": "Table 3 shows GHC fluctuations (100 eps: 14.71, 200: 18.79, 300: 14.56, 400: 25.27). High variance not investigated; why 300 episodes worse than 200 and 400?"
     },
     {
       "flag": "No limitations section or scope boundaries",
       "detail": "Paper frames broadly ('long-horizon agentic tasks', 'real-world') but explicitly bounds only to ALFWorld. No discussion of what approach does NOT work for (e.g., single-turn QA, retrieval, open-ended generation)."
     },
     {
       "flag": "Potential benchmark contamination not addressed",
       "detail": "ALFWorld released 2021; Qwen3 and DeepSeek-R1 likely trained 2024–2025. Possibility of test-set overlap in base model training not tested or discussed."
     },
     {
       "flag": "No code or training data release",
       "detail": "Reproduction impossible without code. The 200 high-precision rollouts for KL-ST training are not released. Evaluation uses public ALFWorld but training is unreproducible."
     }
   ],
   "cited_papers": [
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "relevance": "Early work on agentic language model scaffolding and trajectory-level optimization; foundational for multi-step reasoning."
     },
     {
       "title": "ALFWorld: Aligning text and embodied environments for interactive learning",
       "relevance": "Primary evaluation benchmark; establishes the agentic task domain (household manipulation via natural language)."
     },
     {
       "title": "ReAct: Synergizing reasoning and acting in language models",
       "relevance": "Core agentic framework combining reasoning and tool use; baseline for step-level decision-making architectures."
     },
     {
       "title": "RouteLLM: Learning to route llms with preference data",
       "relevance": "Prior routing work at query level; this paper advances to step-level routing. Direct predecessor in routing literature."
     },
     {
       "title": "Can compressed llms truly act? an empirical evaluation of agentic capabilities in LLM compression",
       "relevance": "Directly relevant: quantization causes agentic failure on interactive tasks, motivating the precision-routing approach."
     },
     {
       "title": "BEST-Route: Adaptive LLM routing with test-time optimal compute",
       "relevance": "Concurrent work on compute-aware routing; establishes the importance of cost-benefit trade-offs in routing."
     },
     {
       "title": "Quantization hurts reasoning? an empirical study on quantized reasoning models",
       "relevance": "Shows quantization degrades performance on reasoning-heavy tasks; supports motivation for adaptive precision selection."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "relevance": "Alternative agentic benchmark (software engineering); highlights generalization question beyond ALFWorld."
     },
     {
       "title": "WebArena: A realistic web environment for building autonomous agents",
       "relevance": "Another agentic benchmark domain (web navigation); would provide generalization evidence if evaluated here."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Addresses real cost-efficiency problem for deployed agentic systems, but requires retraining router per model/benchmark pair and depends on GPTQ quantization availability. Limited immediate applicability."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "Finding that quantized models fail at 'critical steps' is intuitive (confirms practitioner belief). Step-level routing is incremental advance over query-level. No surprising or contrarian insight."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No discussion of safety, alignment, or AI risk. Paper is purely about efficiency optimization."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "Straightforward technical contribution; no controversy, heated debates, or conflicting findings."
     },
     "demo_ability": {
       "score": 1,
       "justification": "No code release, so demo requires reimplementation. Figure 1 case study is illustrative but no runnable demo provided. Difficult to try immediately."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Authors from University of Arizona, Pittsburgh, UNC Chapel Hill, UCF—solid institutions but not top-tier ML labs (no Anthropic, OpenAI, DeepMind, Meta affiliations)."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }