ai-research-survey

scan.json (26484B)

---

 {
   "paper": {
     "title": "PersonalizedRouter: Personalized LLM Routing via Graph-based User Preference Modeling",
     "authors": ["Zhongjie Dai", "Tao Feng", "Jiaxuan You"],
     "year": 2025,
     "venue": "Transactions on Machine Learning Research",
     "arxiv_id": "2511.16883",
     "doi": "10.48550/arXiv.2511.16883"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "PersonalizedRouter, a graph-based framework for personalized LLM routing, outperforms existing LLM selection baselines by 9.83-15.38% on small-scale experiments and 16.19-59.69% on a 1,000-user benchmark (PersonaRoute-Bench). The framework demonstrates few-shot generalization to new users and new LLMs, achieving 64.81-96.01% and 85.90% of fully trained performance respectively. A small real-user validation (40 users) shows 6.05% improvement over the best baseline, though the evaluation relies heavily on simulated users with fixed preferences.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a GitHub repository link (ulab-uiuc/PersonalizedRouter) and a Hugging Face Collection link in the abstract section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper references a Hugging Face Collection for PersonaRoute-Bench, and the source datasets (Alpaca, GSM8K, SQuAD, Multi-News) are all publicly available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.4 mentions PyTorch and PyG on an NVIDIA A6000 48GB GPU, but no version numbers for frameworks, no requirements.txt, and no environment specification file."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides implementation details (Section 4.4) and a code link, but no step-by-step reproduction instructions, no commands to run, and no 'Reproducing Results' section in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 3-8 are reported as single point estimates with no confidence intervals, error bars, or ± notation."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements (e.g., 15.38%, 9.83%) by comparing raw numbers without any statistical significance tests (no p-values, t-tests, or other tests)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables report both absolute scores and percentage improvements with baseline context (e.g., PersonalizedRouter 0.255 vs GraphRouter 0.221, 15.38% improvement in Table 3), providing enough context to judge effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 600 cases per dataset, 9 users in small-scale, 1,000 simulated users in large-scale, or 40 real users. No power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviation, variance, or any spread measure reported across experimental runs. All results appear to be single-run numbers."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: HybridLLM, FrugalGPT, GraphRouter, RouterDC, plus an Oracle upper bound (Section 4.3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include GraphRouter (2024), RouterDC (2024), HybridLLM (2024), and FrugalGPT (2023), all recent and representative of the state of the art in LLM routing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 presents an ablation of GNN depth (0-5 layers), showing that the GNN component is necessary (0 layers performs worst) and 2-3 layers is optimal."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two main metrics are used: Reward score (for multi-cost-efficiency simulation) and Accuracy (for LLM-as-a-Judge strategy). Table 4 also reports time cost."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.5 includes a Human-as-a-Judge scenario with 40 real users selecting preferred LLM responses across 80 queries, validating the router against real user preferences."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes a 70:10:20 train/validation/test split, with results reported on the held-out test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as aggregate scores per method. No per-task-dataset breakdown (e.g., performance on Alpaca vs GSM8K vs SQuAD vs Multi-News separately) is provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No error analysis or qualitative examples of routing failures. The paper does not discuss specific cases where PersonalizedRouter made incorrect routing decisions or why."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.6 reports that deeper GNN layers (>3) degrade performance due to over-smoothing. Table 6 shows that the few-shot new-user model underperforms the best baseline by -30.69% under multi-cost-efficiency."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 15.38% and 9.83% improvement are supported by Table 3. Claims of 16.19% and 59.69% on PersonaRoute-Bench are supported by Table 4. Few-shot generalization numbers match Tables 6-7."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's causal claims about GNN depth (Section 5.6) use controlled ablation. The main comparative claims are supported by controlled experiments on the same data with the same evaluation protocol."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'strong generalization to real-user scenarios' (Section 5.5) based on only 40 users with 80 queries. The title implies general 'Personalized LLM Routing' but evaluation is limited to 4 task types, simulated preferences, and a narrow set of candidate LLMs."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for PersonalizedRouter's improvements. It does not consider whether the gains come from the graph structure vs. heterogeneous node features vs. simply having user-specific parameters."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures routing accuracy on simulated user preferences (reward scores and LLM-judge labels) and frames this as 'personalized LLM selection' for real users. The gap between simulated preferences and actual user satisfaction is acknowledged only briefly in the limitations section."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Candidate LLMs are listed by family and size (e.g., 'LLaMA-3 (8B)', 'Mixtral-8x7B') without exact versions or snapshot dates. Judge models ('DeepSeek-V3', 'Kimi k2', 'Llama-3.3 70B') also lack version specifics."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "LLM judge instruction prompts are provided in full in Tables 10-11 (Appendix A.3), and user profiles used as system prompts are listed in Table 12 (Appendix A.4)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "GNN training hyperparameters are well-reported (Section 4.4: 2-layer GAT, hidden dim 32, batch 32, 400 epochs, Adam, LambdaLR from 1e-3 to 0), but LLM API settings (temperature, top-p, max tokens) for generating candidate responses are not stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use agentic scaffolding. PersonalizedRouter is a routing framework, not an agentic system."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 documents the interaction dataset construction: uniform sampling of 600 cases per dataset, LLM response collection, reward calculation formula, judge labeling procedure, and train/val/test splitting (70:10:20)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 is titled 'Limitations' and discusses the gap between simulated and real user behavior and the lack of an explicit utility function."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "Section 7 states 'user behavior is often more complex, potentially involving a mixture of preferences and evolving over time' and suggests future work on utility functions. These are generic observations rather than specific threats to the validity of the current results."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what its results do NOT show. No specific exclusions are listed regarding task types, LLM categories, or user populations where the approach may fail."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a Hugging Face Collection link for PersonaRoute-Bench and a GitHub repository link. Source datasets (Alpaca, GSM8K, SQuAD, Multi-News) are all publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2 describes the data collection process: sampling queries from 4 datasets, generating LLM responses via API, computing reward scores or collecting judge labels, with formulas and procedures documented."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "For the human study (Section 5.5), the paper states only 'we recruited 40 users' without describing recruitment channels, eligibility criteria, or potential selection bias."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "While the overall construction procedure is described, specific filtering steps and counts are missing. The paper does not explain the sampling procedure for selecting 600 cases from each dataset or whether any LLM responses were excluded."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section states: 'We sincerely appreciate the research gift from Lenovo that made this project possible.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are affiliated with the University of Illinois at Urbana-Champaign, clearly stated in the paper header. No conflict with the evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Lenovo provided a research gift but has no direct financial stake in which LLM router performs best. No Lenovo products are being evaluated in the experiments."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the candidate LLMs (LLaMA-3, Mixtral, Qwen, etc.) used to generate responses on benchmark datasets."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether candidate LLMs were trained on the evaluation datasets (GSM8K published 2021, SQuAD published 2016 — both likely in training data of modern LLMs)."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021) and SQuAD (2016) are well-known benchmarks available online before the training of all candidate LLMs. No contamination analysis or discussion is provided, though contamination could distort the interaction data quality."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": true,
         "answer": false,
         "justification": "No mention of pre-registration for the 40-user human study in Section 5.5."
       },
       "irb_or_ethics_approval": {
         "applies": true,
         "answer": false,
         "justification": "No mention of IRB or ethics board approval for the human study involving 40 participants."
       },
       "demographics_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '40 users' without reporting any demographic information (experience level, background, age, etc.)."
       },
       "inclusion_exclusion_criteria": {
         "applies": true,
         "answer": false,
         "justification": "No inclusion or exclusion criteria are stated for the 40 human participants. The paper says only 'we recruited 40 users.'"
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a preference elicitation study where users select their preferred response, not an experiment with random assignment to treatment conditions."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "This is a preference selection study, not a blinded experiment. Users see all 10 LLM responses without condition assignments to blind."
       },
       "attrition_reported": {
         "applies": true,
         "answer": false,
         "justification": "No information on how many of the 40 recruited users completed the full questionnaire or whether any responses were excluded."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports time costs for each method (e.g., PersonalizedRouter: 10:15 under multi-cost-efficiency, 11:37 under LLM-as-a-Judge) with reduction percentages. LLM cost per million tokens is listed in Tables 16-18."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using an NVIDIA A6000 48GB GPU (Section 4.4) but does not state total GPU hours, total API spend, or aggregate computational budget across all experiments."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No results across multiple random seeds are reported. All results appear to be from single runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs producing the reported results is never stated anywhere in the paper."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Final hyperparameters are reported (Section 4.4) but no hyperparameter search budget, search method, or number of configurations tried is disclosed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "GNN depth is justified via ablation (Figure 2), but other hyperparameter choices (hidden dim 32, batch size 32, learning rate 1e-3) are not justified or shown to be selected on a validation set."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across methods and settings without any statistical tests at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GraphRouter, a key baseline, is prior work by the same research group (Feng et al., 2024b, same PI Jiaxuan You). The bias of evaluating their own system against their own prior baseline is not acknowledged."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 reports both performance (Reward/Accuracy) and time cost with reduction percentages for all methods, allowing performance-at-compute comparison. PersonalizedRouter achieves 96.63% time reduction vs RouterDC."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "PersonaRoute-Bench uses simulated users with fixed α/β weight pairs or LLM-generated profiles. The paper does not discuss whether these simulated preferences adequately measure real-world personalized routing capability."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved in the routing evaluation. The router directly predicts LLM assignments without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether candidate LLMs' training data includes the benchmark datasets used for evaluation (GSM8K from 2021, SQuAD from 2016)."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information, such as interaction data structure embedding answer-relevant information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential non-independence between training and test interaction data, which are drawn from the same users and datasets with random splitting."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied in any of the experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "PersonalizedRouter outperforms existing LLM selection methods by 15.38% (multi-cost-efficiency) and 9.83% (LLM-as-a-Judge) in the standard setting with 10 LLMs and 9 users.",
       "evidence": "Table 3 (Section 5.1): Reward 0.255 vs GraphRouter 0.221 (15.38%); Accuracy 0.447 vs RouterDC 0.407 (9.83%).",
       "supported": "moderate"
     },
     {
       "claim": "On PersonaRoute-Bench with 1,000 users, PersonalizedRouter surpasses the best baseline by 16.19% and 59.69% under two simulation strategies.",
       "evidence": "Table 4 (Section 5.2): Reward 0.244 vs RouterDC 0.210 (16.19%); Accuracy 0.313 vs HybridLLM 0.196 (59.69%).",
       "supported": "moderate"
     },
     {
       "claim": "PersonalizedRouter demonstrates few-shot generalization, achieving 64.81% and 85.90% of the fully trained model's performance when adapting to new users and new LLMs.",
       "evidence": "Table 6 (Section 5.3): new users multi-cost 0.07/0.108 = 64.81%. Table 7 (Section 5.4): new LLMs multi-cost 0.201/0.234 = 85.90%.",
       "supported": "moderate"
     },
     {
       "claim": "PersonalizedRouter outperforms the best baseline by 6.05% on a small-scale real-user dataset with 40 users.",
       "evidence": "Table 8 (Section 5.5): Accuracy 0.438 vs RouterDC 0.413 (6.05%). Only 40 users with 80 queries.",
       "supported": "weak"
     },
     {
       "claim": "GNN depth of 2-3 layers is optimal; deeper networks suffer from over-smoothing that degrades prediction performance.",
       "evidence": "Figure 2 (Section 5.6): performance peaks at 2-3 layers and declines with 4-5 layers under both simulation strategies.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Simulated user evaluation",
       "detail": "The vast majority of evaluation uses simulated users with fixed preference parameters (α/β weight pairs or LLM-generated profiles). The real-user validation has only 40 users with 80 queries — too small to support claims of 'strong generalization to real-user scenarios.'"
     },
     {
       "flag": "No error bars or significance tests",
       "detail": "All results are single-run point estimates with no uncertainty quantification. The claimed improvements (e.g., 15.38%, 9.83%) could be within random noise, particularly given the small scale of some experiments."
     },
     {
       "flag": "Self-comparison with prior work from same group",
       "detail": "GraphRouter, a key baseline, is prior work by the same research group (same PI Jiaxuan You). The authors implement both their own method and this baseline, introducing potential self-comparison bias per Lucic et al. (2018)."
     },
     {
       "flag": "Selective framing of few-shot results",
       "detail": "The abstract reports 64.81% for new-user generalization (multi-cost-efficiency), but under the same setting, the few-shot model underperforms the best baseline by -30.69% (Table 6). The LLM-as-a-Judge setting shows 96.01%, which is much stronger but not highlighted in the abstract."
     },
     {
       "flag": "No contamination discussion for standard benchmarks",
       "detail": "GSM8K (2021) and SQuAD (2016) are well-known benchmarks likely present in the training data of modern LLMs. Contamination could affect LLM performance scores used to construct interaction data, but this is never discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Foundational LLM cost-optimization routing method using response scoring under cost budgets."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang"],
       "year": 2024,
       "arxiv_id": "2404.14618",
       "relevance": "Binary LLM routing method balancing cost and quality, key baseline for LLM selection research."
     },
     {
       "title": "GraphRouter: A Graph-based Router for LLM Selections",
       "authors": ["Tao Feng", "Yanzhen Shen", "Jiaxuan You"],
       "year": 2024,
       "relevance": "GNN-based LLM selection using heterogeneous graphs, direct predecessor to PersonalizedRouter."
     },
     {
       "title": "RouterDC: Query-based Router by Dual Contrastive Learning for Assembling Large Language Models",
       "authors": ["Chen Shuhao", "Jiang Weisen", "Lin Baijiong"],
       "year": 2024,
       "relevance": "Embedding-based LLM routing using cosine similarity, competitive baseline in LLM selection."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2024,
       "arxiv_id": "2406.18665",
       "relevance": "LLM routing using real user preference data from Chatbot Arena with matrix factorization."
     },
     {
       "title": "PolyRouter: A Multi-LLM Querying System",
       "authors": ["Dimitris Stripelis", "Zijian Hu", "Jipeng Zhang"],
       "year": 2024,
       "arxiv_id": "2408.12320",
       "relevance": "Explores multiple routing strategies (KNN, MLP) for multi-LLM query assignment."
     },
     {
       "title": "Cost-effective Online Multi-LLM Selection with Versatile Reward Models",
       "authors": ["Xiangxiang Dai", "Jin Li", "Xutong Liu"],
       "year": 2024,
       "arxiv_id": "2405.16587",
       "relevance": "Bandit-based LLM routing with exploration-exploitation balance for online LLM selection."
     },
     {
       "title": "DeepSeek-V3 Technical Report",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2412.19437",
       "relevance": "Major open LLM technical report; used as judge model in PersonalizedRouter evaluation."
     },
     {
       "title": "PREMIUM: LLM Personalization with Individual-Level Preference Feedback",
       "authors": ["Yihang Sun", "Tao Feng", "Ge Liu", "Jiaxuan You"],
       "year": 2025,
       "relevance": "Related work on LLM personalization through individual preference modeling from the same research group."
     },
     {
       "title": "FusionFactory: Fusing LLM Capabilities with Multi-LLM Log Data",
       "authors": ["Tao Feng", "Haozhen Zhang", "Zijie Lei"],
       "year": 2025,
       "arxiv_id": "2507.10540",
       "relevance": "Multi-LLM capability fusion approach from the same group, related to LLM selection and routing."
     }
   ]
 }