ai-research-survey

scan.json (28604B)

---

 {
   "paper": {
     "title": "HybridFlow: Resource-Adaptive Subtask Routing for Efficient Edge-Cloud LLM Inference",
     "authors": [
       "Jiangwen Dong",
       "Jiayu Li",
       "Tianhang Zheng",
       "Wanyu Lin"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2512.22137"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "HybridFlow decomposes complex queries into dependency-aware DAGs and routes subtasks between edge and cloud models using a learned utility predictor with adaptive budget-aware thresholds. On GPQA, MMLU-Pro, AIME24, and LiveBench-Reasoning, it achieves 55.34% average accuracy (competitive with cloud-only CoT at 58.99%) while reducing latency by 28.5% and API cost by 31.3% versus the best collaborative baseline (DoT). The adaptive router substantially outperforms any fixed-threshold policy (utility 0.7940 vs peak 0.6329), and a model-pair swap experiment suggests transferability beyond the primary Llama3.2-3B/GPT-4.1 configuration.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code: HybridFlow' which appears to be a hyperlinked repository reference. A code artifact is indicated as available."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation benchmarks (GPQA, MMLU-Pro, AIME24, LiveBench-Reasoning) are standard public datasets. Router training uses MMLU-Pro and Math500, also public."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single NVIDIA RTX 3090 GPU' and model names but provides no requirements.txt, Dockerfile, or detailed library version listing sufficient to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper or appendix. The implementation details section describes the system but does not include commands or scripts to replicate experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report mean ± std for all metrics across all benchmarks (e.g., '53.33±2.03' for GPQA accuracy)."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims HybridFlow outperforms baselines based on comparing point estimates with std. No statistical significance tests (t-tests, bootstrap tests, etc.) are performed despite multiple comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with absolute values and baseline context throughout, e.g., 'HybridFlow achieves an average Ctime of 17.48 s, outperforming HybridLLM (24.45 s)' and accuracy comparisons with both numbers provided."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for benchmark sizes or number of profiling queries (2,000). No power analysis or discussion of whether sample sizes are adequate for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported across runs in Tables 1 and 2 (e.g., '53.33±2.03', '15.24±0.30')."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Six baselines are included: Direct Prompt, CoT, SoT, PASTA, HybridLLM, and DoT. Both single-model and collaborative paradigms are represented."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include PASTA (2025), DoT (2025), HybridLLM (2024), and SoT (2024). These are recent and representative of the state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.3 provides ablation studies: Edge-only, Cloud-only, Random routing, Fixed Threshold, and HybridFlow-Chain (disabling DAG parallelism). Table 3 isolates routing vs scheduling contributions."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Four metrics are used: Accuracy, Ctime (latency), CAPI (API cost), and Utility (unified benefit-cost ratio). Tables 1 and 2 report all four."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via benchmark answer matching. The paper makes claims about 'reasoning quality' but relies solely on automated accuracy metrics."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The router is trained on '2,000 sampled queries drawn from two benchmarks: MMLU-Pro (different from the main test samples) and Math500' (Appendix C). Evaluation benchmarks include GPQA and AIME24 which are not in the training set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per benchmark (GPQA, MMLU-Pro, AIME24, LiveBench-Reasoning) in Tables 1 and 2, and per subtask position in Figure 3."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure case analysis is provided. Table 5 shows 9-10% of queries fall back to chain plans, but the impact on accuracy for these cases is not discussed. No qualitative error analysis."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 shows HybridFlow-Chain achieves lower accuracy (50.62%) and utility (0.6095) than full HybridFlow (53.33%, 0.7940), and Table 6 shows how aggressive thresholds degrade performance. These demonstrate configurations that don't work well."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims HybridFlow 'improves the cost-accuracy trade-off, reducing latency and cloud API usage while maintaining competitive accuracy.' Tables 1-2 confirm: 55.34% avg accuracy with 17.48s latency and 0.0088 CAPI, outperforming collaborative baselines on all three dimensions."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims like 'parallel execution substantially reduces end-to-end latency' and 'Router is crucial for achieving a balance' are supported by controlled ablations in Table 3 (e.g., HybridFlow-Chain removes DAG parallelism; Edge/Cloud/Random remove router). Single-variable manipulation in ablations is adequate."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Edge-Cloud LLM Inference' broadly, but results are primarily on one edge/cloud model pair (Llama3.2-3B/GPT-4.1) across four reasoning benchmarks. The conclusion states 'the promise of our adaptive, parallel-aware routing framework for orchestrating efficient edge-cloud AI' without bounding to the tested setting. One model-swap experiment (Appendix D.2) is acknowledged as limited."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of alternative explanations for the results. For example, gains could be partially due to the specific model pair characteristics, benchmark properties, or the planner's decomposition style. No robustness checks beyond the single model-swap experiment."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper frames benchmark accuracy as 'reasoning quality' (e.g., 'preserving reasoning quality' in abstract, 'reasoning quality and system efficiency' in Sec 4.1) without discussing the gap between automated benchmark correctness and actual reasoning quality."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are named as 'Llama3.2-3B', 'GPT-4.1', 'qwen3-embedding-0.6b', 'Qwen2.5-7B', 'DeepSeek-V3'. GPT-4.1 lacks a snapshot date or API version. Per schema requirements, marketing names without version identifiers do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The full planner prompt is provided in Figure 6 with complete text, including XML format constraints, examples, and EAG structure. Figure 7 shows a complete case study with actual subtask descriptions and outputs."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature 0.6, AdamW learning rate 1e-4, τ0=0.2, Kmax=0.02, Lmax=20, ε=10^-4, nmax=7, Rmax=2, normalization scales 10 (latency) and 0.02 (API cost)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The entire scaffolding is described in detail: DAG decomposition pipeline, validation and repair procedure, scheduler queue, router mechanism with utility prediction and adaptive thresholding, and online calibration via contextual bandits. Algorithm 1 formalizes the full procedure."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the profiling data pipeline: 2,000 queries from MMLU-Pro and Math500, paired edge/cloud executions, cached outputs recombined into mixed routing vectors, marginal effect estimation by toggling subtask assignments. Normalization constants and utility target computation are specified."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations section exists. The conclusion makes no mention of limitations. Appendix D.2 briefly notes 'broader coverage of model families is left for future work' but this is not a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper. There is no analysis of specific threats such as planner failure rates affecting results, model-pair specificity, or benchmark selection bias."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not specify what the results do NOT show (e.g., not tested on generation tasks, not tested with models of other scales, not tested under real network variability)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. Per-query results, profiling data, router training data, and detailed execution logs are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C describes profiling data collection: 2,000 queries from MMLU-Pro and Math500, paired execution on edge and cloud, evaluation via task-specific verifiers, and marginal correctness computation via reuse-and-recombine strategy."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C documents the full pipeline: query decomposition → paired edge/cloud execution → cached output recombination → marginal effect estimation → utility target computation → router training with MSE loss. Normalization and clipping procedures are specified."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed anywhere in the paper. There is no acknowledgments section mentioning grants or sponsors."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Hong Kong Polytechnic University (Departments of Data Science and AI, and Computing) and Zhejiang University. None are affiliated with the evaluated models' companies."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence of funder cannot be assessed. The paper evaluates GPT-4.1 (OpenAI) and Llama3.2-3B (Meta) — without a funding statement, potential conflicts remain unknown."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for GPT-4.1, Llama3.2-3B, or any model used. This is relevant since GPQA and MMLU-Pro could be in the training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPT-4.1 or Llama3.2-3B were trained on GPQA, MMLU-Pro, AIME24, or LiveBench-Reasoning data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GPQA, MMLU-Pro, and AIME24 were all published before GPT-4.1's likely training cutoff. LiveBench is designed to mitigate contamination. No contamination discussion is provided for any benchmark."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables 1 and 2 report per-query latency (Ctime in seconds) and API cost (CAPI in dollars) for all methods across all benchmarks. Cost is a central evaluation metric."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'single NVIDIA RTX 3090 GPU' for edge computation but does not state total GPU hours, training time for the router, or total API spend for profiling the 2,000 training queries."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Tables report mean ± std but do not explicitly discuss random seed sensitivity. It is unclear whether variance comes from different seeds, sampling randomness at temperature 0.6, or other sources."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as 'mean ± std' (Table 1 caption) but the exact number of runs is never stated."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Hyperparameters are described as 'empirically set ... based on preliminary tuning across all benchmarks' (Appendix C) but the number of configurations tried or compute spent on tuning is not reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "While Table 6 shows a sweep of fixed thresholds and the optimization framework motivates adaptive thresholding, the final configuration (τ0=0.2, Kmax=0.02, Lmax=20) is 'empirically set' without explaining the selection criterion or validation procedure."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple methods are compared across multiple benchmarks with no correction for multiple comparisons and no significance tests at all."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own system against their reimplementations of baselines. No acknowledgment of potential author-evaluation bias per Lucic et al. (2018)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance is explicitly plotted as a function of compute budget. Figure 4 shows accuracy, cost, and utility across threshold values. Table 6 provides detailed cost-performance data. The Utility metric (Eq. 2) directly captures performance per unit cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GPQA, MMLU-Pro, AIME24, or LiveBench-Reasoning actually measure the 'reasoning quality' and 'edge-cloud inference efficiency' the paper claims to evaluate. Benchmarks are used without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Main comparisons mix scaffolding and routing differences: HybridFlow (DAG + adaptive routing) vs DoT (sequential + routing) vs HybridLLM (query-level routing). The scaffold effect is not isolated from the routing effect in the main results. The ablation (Table 3, HybridFlow-Chain) partially addresses this but the main cross-method comparisons do not."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. GPT-4.1 may have been trained on data that includes solutions to GPQA, MMLU-Pro, or AIME24 problems."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The DAG decomposition by the planner could provide structural hints that wouldn't be available in standard inference, but this is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The router training uses MMLU-Pro samples 'different from the main test samples' showing some awareness, but no formal independence verification. No discussion of LLM training data overlap with benchmarks."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used (no canary strings, membership inference, n-gram overlap analysis, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "HybridFlow achieves competitive accuracy (55.34% avg) while reducing latency and API cost compared to collaborative baselines.",
       "evidence": "Table 1 shows 55.34% avg accuracy vs DoT 46.50% and HybridLLM 38.70%. Table 2 shows Ctime 17.48s vs HybridLLM 24.45s and DoT 18.32s; CAPI 0.0088 vs HybridLLM 0.0128 and DoT 0.009.",
       "supported": "strong"
     },
     {
       "claim": "Dependency-aware DAG planning enables parallelism that substantially reduces end-to-end latency.",
       "evidence": "Table 3 ablation: HybridFlow achieves 15.24s latency and 0.7940 utility vs HybridFlow-Chain (no DAG parallelism) at 16.12s and 0.6095 utility on GPQA. Latency improvement from DAG parallelism is modest (~0.88s).",
       "supported": "moderate"
     },
     {
       "claim": "The adaptive routing mechanism outperforms any fixed-threshold policy.",
       "evidence": "Table 3 shows adaptive routing utility 0.7940 vs best fixed threshold (τ0=0.6) utility 0.6329 in Table 6. Table 6 provides a comprehensive threshold sweep from τ0=0 to τ0=1.",
       "supported": "strong"
     },
     {
       "claim": "HybridFlow's design generalizes beyond the Llama3.2-3B/GPT-4.1 model pair.",
       "evidence": "Table 8 shows one model-swap experiment (Qwen2.5-7B/DeepSeek-V3) on GPQA only. HybridFlow maintains advantages (53% accuracy, $1.16e-3 API cost) vs DoT (49%, $1.80e-3) under the swapped pair.",
       "supported": "weak"
     },
     {
       "claim": "HybridFlow's router concentrates cloud usage on early, high-impact subtasks and shifts to edge for later subtasks.",
       "evidence": "Figure 3 shows position-dependent offloading pattern on GPQA with cloud calls decreasing and adaptive threshold increasing at later subtask positions.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No limitations section",
       "detail": "The paper contains no limitations, threats-to-validity, or scope-boundaries discussion. For a systems paper making broad claims about edge-cloud LLM inference, the absence of any limitations acknowledgment is a significant omission."
     },
     {
       "flag": "Single primary model pair",
       "detail": "Main results use only Llama3.2-3B/GPT-4.1. The single model-swap experiment (Qwen2.5-7B/DeepSeek-V3) is limited to GPQA only. Claims about general edge-cloud inference are weakly supported by this narrow evaluation."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Multiple comparative claims are made by comparing point estimates with standard deviations, but no significance tests are performed. Several results overlap in their error ranges (e.g., HybridFlow 53.33±2.03 vs HybridLLM 52.9±0.94 on GPQA)."
     },
     {
       "flag": "Planner failure rate not analyzed",
       "detail": "Table 5 shows 9-10% of queries fall back to chain plans (sequential execution), yet the impact on accuracy and efficiency for these fallback cases is not reported separately. This could mask systematic failures on certain query types."
     },
     {
       "flag": "Number of experimental runs not stated",
       "detail": "Results are reported as mean ± std but the number of runs is never specified. The reader cannot assess whether variance estimates are reliable."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "GPT-4.1 may have been trained on GPQA, MMLU-Pro, and AIME24 benchmark data. Since HybridFlow routes subtasks to GPT-4.1, contamination could inflate accuracy results for cloud-routed subtasks, making the routing appear more effective than it is."
     }
   ],
   "cited_papers": [
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["Dujian Ding", "Ankur Mallick", "Chi Wang", "Robert Sim", "Subhabrata Mukherjee", "Victor Rühle", "Laks V. S. Lakshmanan", "Ahmed H. Awadallah"],
       "year": 2024,
       "relevance": "Cost-efficient LLM query routing between small and large models, directly comparable baseline for edge-cloud collaboration."
     },
     {
       "title": "Division-of-Thoughts: Harnessing Hybrid Language Model Synergy for Efficient On-Device Agents",
       "authors": ["Chenhan Shao", "Xiaohan Hu", "Yilu Lin", "Fangkai Xu"],
       "year": 2025,
       "relevance": "Hybrid language model collaboration for on-device agents with sequential reasoning delegation."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Cost-aware LLM cascade strategies for budget-quality trade-offs in LLM inference."
     },
     {
       "title": "SplitReason: Learning to Offload Reasoning",
       "authors": ["Yash Akhauri", "Andy Fei", "Chia-Chih Chang", "Amr F. AbouElhamayed", "Yejin Li", "Mohamed S. Abdelfattah"],
       "year": 2025,
       "arxiv_id": "2504.16379",
       "relevance": "Fine-grained reasoning step offloading from edge to cloud models, closely related to subtask-level routing."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "authors": ["Jason Wei", "Xuezhi Wang", "Dale Schuurmans", "Maarten Bosma", "Fei Xia", "Ed Chi", "Quoc V. Le", "Denny Zhou"],
       "year": 2022,
       "relevance": "Foundational work on chain-of-thought reasoning that HybridFlow extends with parallel decomposition and routing."
     },
     {
       "title": "Skeleton-of-Thought: Prompting LLMs for Efficient Parallel Generation",
       "authors": ["Xuefei Ning", "Zinan Lin", "Zixuan Zhou", "Zifu Wang", "Huazhong Yang", "Yu Wang"],
       "year": 2024,
       "relevance": "Parallel LLM generation via skeleton decomposition, one of the main baselines for parallel inference."
     },
     {
       "title": "Graph of Thoughts: Solving Elaborate Problems with Large Language Models",
       "authors": ["Maciej Besta", "Nils Blach", "Ales Kubicek"],
       "year": 2024,
       "relevance": "Graph-based reasoning with LLMs enabling flexible dependency patterns, precursor to DAG-based decomposition."
     },
     {
       "title": "Tree of Thoughts: Deliberate Problem Solving with Large Language Models",
       "authors": ["Shunyu Yao", "Dian Yu", "Jeffrey Zhao"],
       "year": 2023,
       "relevance": "Tree-structured LLM reasoning with deliberate search, influential structured inference approach."
     },
     {
       "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark",
       "authors": ["David Rein", "Betty Li Hou", "Asa Cooper Stickland"],
       "year": 2024,
       "relevance": "Graduate-level reasoning benchmark used as primary evaluation dataset."
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
       "authors": ["Yubo Wang", "Xueguang Ma", "Ge Zhang"],
       "year": 2024,
       "arxiv_id": "2406.01574",
       "relevance": "Enhanced multi-task benchmark for LLM evaluation, used as evaluation and router training data source."
     },
     {
       "title": "LiveBench: A Challenging, Contamination-Limited LLM Benchmark",
       "authors": ["Colin White", "Samuel Dooley", "Manley Roberts"],
       "year": 2025,
       "relevance": "Contamination-resistant benchmark designed to provide clean LLM evaluation signals."
     },
     {
       "title": "S-DAG: A Subject-Based Directed Acyclic Graph for Multi-Agent Heterogeneous Reasoning",
       "authors": ["Jiangwen Dong", "Zhiqi Lin", "Wanyu Lin", "Mingxuan Zhang"],
       "year": 2025,
       "arxiv_id": "2511.06727",
       "relevance": "DAG-based multi-agent reasoning framework by same first author, precursor to HybridFlow's decomposition approach."
     }
   ]
 }