ai-research-survey

scan-v5.json (23930B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "HAPS: Hierarchical LLM Routing with Joint Architecture and Parameter Search",
     "authors": [
       "Zihang Tian",
       "Rui Li",
       "Jingsen Zhang",
       "Xiaohe Bo",
       "Wei Huo"
     ],
     "year": 2026,
     "venue": "arXiv.org",
     "arxiv_id": "2601.05903",
     "doi": "10.48550/arXiv.2601.05903"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims HAPS 'consistently outperforms strong routing baselines' on two benchmarks; Table 2 confirms best performance in 5 of 6 open-source pair settings, and code is released at the stated GitHub URL.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Ablation studies (Tables 3, Figure 3, Figure 4) systematically isolate the high-level router, low-level router, and parameter sharing, providing adequate within-system causal evidence for each component's contribution.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion calls HAPS 'a promising and practical direction for building efficient, adaptive AI systems' based on 100 test instances across two benchmarks and pairwise model configurations; the scope of the claim exceeds what these narrow evaluations can support.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not consider whether improvements stem from LoRA fine-tuning on the training distribution rather than routing per se, nor whether the Teacher-Student scaffold itself drives gains independent of routing.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "Token-level F1 for HotpotQA and exact-match accuracy for MMLU are used as direct measures of the claimed 'task performance'; the paper does not conflate these with broader capability claims.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section is present at the end of the main paper listing four distinct limitations.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Three of the four stated limitations are generic ('effectiveness may vary when task distribution changes'; 'reward signals can be noisy'; 'reproducibility may be limited'); none name specific confounds such as the 100-instance test set size or distribution shift from exhaustive enumeration training.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations note the method was evaluated on 'a specific set of tasks, model candidates, and cost settings' but do not explicitly bound what the results do NOT show (e.g., no claims about multi-hop reasoning beyond 2-agent pairs or non-pairwise routing pools).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgment or grant number is present anywhere in the paper; only institutional affiliations are listed.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: four authors at Renmin University of China and Wei Huo at Huawei Technologies Wireless Technology Lab.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "One author (Wei Huo) is from Huawei Technologies, an organization with commercial interest in efficient LLM deployment; no independence statement or COI disclosure is made.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "There is no competing interests or financial disclosure statement anywhere in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "LLM routing is formally defined in Section 2 with a mathematical objective; the distinction between 'architecture selection' (discrete) and 'parameter search' (continuous) is clearly explained with a motivating example in Figure 1.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states three contributions: the idea of joint architecture+parameter search, the hierarchical implementation with parameter generation network, and the experimental validation.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 engages with RouteLLM, GraphRouter, IRT-Router, cascade systems, and PEFT literature, explaining how HAPS bridges discrete routing and continuous adaptation in a way prior work does not.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The abstract states 'We have released our code at https://github.com/zihangtian/HAPS', indicating an actual public release.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "Both evaluation datasets (HotpotQA and MMLU) are standard public benchmarks used unmodified.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "No requirements.txt, Dockerfile, or dependency specification is provided; the paper describes model architectures but not the software environment needed to run them.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": true,
           "justification": "Appendices B–D provide detailed Algorithms 1–3, full agent prompts (Figures 6–14), data split construction, and baseline adaptation procedures sufficient to reproduce the pipeline alongside the released code.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "All results in Tables 2, 3, and 5 and Figures 3–5 are single point estimates with no confidence intervals, standard deviations, or error bars across any runs.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are reported for any comparisons; margins as small as 1 F1 point on a 100-instance test set are described as 'consistent' improvements without testing.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Absolute improvement margins are reported throughout (e.g., '1.85%, 3.60%, and 1.63% in F1 score' over the runner-up), providing effect size context.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 100-instance test set is stated in Appendix C without justification or power analysis; no rationale is given for why this size is sufficient for reliable comparisons.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "No variance, standard deviation, or results across multiple runs are reported; it is unclear whether experiments were run once or averaged.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Four baselines are compared: Random, RouteLLM, GraphRouter, and IRT-Router, covering diverse routing paradigms.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "RouteLLM (2024), GraphRouter (2024), and IRT-Router (2025) are recent and represent the state of the art in the LLM routing literature.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Comprehensive ablation covers removal of the high-level router (random or fixed variants), removal of the low-level router, removal of parameter sharing, and variation in LoRA injection depth.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Token-level F1 (HotpotQA) and accuracy (MMLU) are used for quality; normalized cost and a composite reward are used in the trade-off analysis (Section 4.6).",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Automated benchmarks with objective labels (HotpotQA F1, MMLU accuracy) make human evaluation of outputs not applicable here.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C explicitly states Dtest (100 instances) 'is strictly reserved for final evaluation of the complete routing framework' and is not used during training or validation.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": false,
           "justification": "MMLU covers 57 subjects but no per-subject accuracy breakdown is provided; HotpotQA results are also reported only as aggregate F1.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No failure cases or error analysis are presented; the paper only discusses where HAPS performs best, not where it fails.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Figure 4 shows LoRA depth l3 degrades performance for L-Q and L-M pairs relative to l2, reporting the non-monotonic relationship as a finding rather than hiding it.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific versioned model names are given: Llama-3.1-8B-Instruct, Qwen2.5-7B-Instruct, Mistral-7B-Instruct-v0.3, Llama-3.2-1B-Instruct, GPT-4.1 Nano, DeepSeek V3.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full verbatim prompts for all agent roles on both benchmarks are provided in Appendix Figures 6–14, including system prompts and all template fields.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": false,
           "justification": "Only scaling factor α=0.01 and LoRA rank r=8 are reported; learning rates ηsft, ηθ, ηϕ appear as symbols in Algorithm 3 but their values are never given, and training epochs/batch sizes are also absent.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": true,
           "answer": true,
           "justification": "The two-agent Teacher-Student framework, interaction protocols, action spaces, and context management for both HotpotQA and MMLU are described in detail across Appendices A and B with formal algorithms.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C documents stratified sampling over MMLU subjects, sampling from hotpot_dev_distractor_v1.json for HotpotQA, and the exhaustive enumeration procedure for constructing the action-level training dataset.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "The preprocessed train/val/test splits (3000/1000/100 instances) and the exhaustively-enumerated action-reward dataset are not separately published; availability depends on the code repository.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C describes dataset sourcing, split sizes, stratified sampling for MMLU, and the procedure for constructing the action-level dataset via exhaustive enumeration of routing decisions.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; standard public benchmarks are used.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from raw benchmark sampling through SFT warm-up dataset construction, RL dataset construction, and test set isolation is documented in Appendix C and Algorithm 3.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "The paper evaluates LLMs on MMLU (2020) and HotpotQA (2018), both of which predate and are likely included in training corpora of Llama-3.1, Qwen2.5, and Mistral; no training cutoff is stated for any model.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Potential contamination of MMLU or HotpotQA examples in the base LLMs' pretraining data is not discussed anywhere in the paper.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "Neither benchmark was created after the models' training cutoffs; MMLU (2020) and HotpotQA (2018) are well-known benchmarks highly likely to appear in pretraining data, but this is not acknowledged.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants in this study.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; the ethics section notes no PII is collected.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.6 provides an explicit performance-cost trade-off analysis with normalized costs, pricing tables (Table 4), and cost values for all methods under three budget regimes.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "No GPU hours, training time, or total compute budget for training HAPS is reported anywhere in the paper.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "HAPS achieves state-of-the-art performance in 5 of 6 open-source pair settings on HotpotQA and MMLU",
       "evidence": "Table 2 shows HAPS achieves best F1/accuracy in L-Q, M-Q, L-M HotpotQA and M-Q, L-M MMLU; ties on L-Q MMLU at 79%",
       "supported": "moderate"
     },
     {
       "claim": "Parameter sharing between high-level and low-level routers is necessary, contributing up to 4.33% F1 gain",
       "evidence": "Figure 3 shows decoupled variant (w/o Parameter Sharing) drops HotpotQA F1 by 1.64–4.33% and MMLU accuracy by 3–6% across all pairs",
       "supported": "moderate"
     },
     {
       "claim": "Both high-level architecture routing and low-level parameter generation are individually necessary",
       "evidence": "Table 3 ablations show removing the high-level router (random) drops F1 by up to 15%, and removing the low-level router degrades performance across all settings",
       "supported": "moderate"
     },
     {
       "claim": "HAPS achieves better performance-cost trade-offs than baselines, reducing cost by 58% vs RouteLLM while maintaining comparable F1",
       "evidence": "Table 5 shows HAPS achieves cost 0.0886 vs RouteLLM's 0.2111 in performance-first regime while F1 is 41.94% vs 43.35%",
       "supported": "moderate"
     },
     {
       "claim": "HAPS extends naturally to mixed open/closed-source settings by blocking gradient propagation for proprietary APIs",
       "evidence": "Figure 5 shows HAPS outperforms all baselines on L-G (47.19% F1) and Q-D (58.52% F1) mixed-source pairs on HotpotQA",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "HAPS proposes a hierarchical LLM routing framework that jointly searches discrete model architectures and continuous LoRA parameters, outperforming four routing baselines on HotpotQA (F1) and MMLU (accuracy) across multiple model-pair configurations. Parameter sharing between the high-level routing classifier and low-level parameter generation network is a critical design choice, contributing 1.64–4.33% F1 on HotpotQA when ablated. The framework also achieves favourable performance-cost trade-offs by generating concise routing and reducing token consumption, and extends to mixed open/closed-source settings by blocking gradient flow for proprietary APIs. All improvements are measured on 100-instance test sets without statistical significance testing, limiting confidence in the reported margins.",
   "red_flags": [
     {
       "flag": "Tiny test set",
       "detail": "Test sets contain only 100 instances per benchmark; margins of 1–3 F1 points are reported as 'consistent' improvements without any statistical significance testing or error bars."
     },
     {
       "flag": "No variance reported",
       "detail": "All results are single point estimates; no standard deviation, confidence intervals, or multiple-run variance is reported anywhere in the paper."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "MMLU (2020) and HotpotQA (2018) predate all base models used; the possibility that benchmark examples appear in pretraining data is not discussed."
     },
     {
       "flag": "Missing hyperparameters",
       "detail": "Learning rates ηsft, ηθ, ηϕ appear as symbols in Algorithm 3 but their actual values are never reported; training epochs and batch sizes are also absent."
     },
     {
       "flag": "No per-category breakdown",
       "detail": "MMLU covers 57 subjects but only aggregate accuracy is reported; subgroup variation cannot be assessed."
     },
     {
       "flag": "Huawei co-authorship undisclosed",
       "detail": "One author is from Huawei Technologies, a commercial AI stakeholder, but no funding disclosure or competing interests statement is made."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "relevance": "Primary routing baseline and inspiration for preference-based binary routing"
     },
     {
       "title": "GraphRouter: A Graph-based Router for LLM Selections",
       "relevance": "Heterogeneous graph routing baseline"
     },
     {
       "title": "IRT-Router: Effective and Interpretable Multi-LLM Routing via Item Response Theory",
       "relevance": "Contemporary interpretable routing baseline with difficulty modeling"
     },
     {
       "title": "LoRA: Low-Rank Adaptation of Large Language Models",
       "relevance": "Foundation for the parameter generation component of HAPS"
     },
     {
       "title": "HotpotQA: A Dataset for Diverse, Explainable Multi-Hop Question Answering",
       "relevance": "Primary multi-hop reasoning evaluation benchmark"
     },
     {
       "title": "Measuring Massive Multitask Language Understanding (MMLU)",
       "relevance": "Primary broad-knowledge evaluation benchmark"
     },
     {
       "title": "HyperNetworks",
       "relevance": "Architectural basis for the parameter generation network design"
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "relevance": "Related work on routing evaluation infrastructure"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "LLM routing for cost-performance trade-offs is directly applicable to production deployments, and code is released."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The joint architecture+parameter framing is a modest but not surprising extension of existing routing work."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety or risk concerns are raised; the paper is purely a performance optimization contribution."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflicting claims with established work."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is publicly released on GitHub; practitioners could in principle reproduce the routing setup on public benchmarks."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Huawei affiliation is notable but the paper is not from a marquee AI lab; venue is arXiv preprint."
     }
   },
   "hn_data": {
     "threads": [],
     "top_points": 0,
     "total_points": 0,
     "total_comments": 0
   }
 }