ai-research-survey

scan.json (29989B)

---

 {
   "paper": {
     "title": "Pay for Hints, Not Answers: LLM Shepherding for Cost-Efficient Inference",
     "authors": [
       "Ziming Dong",
       "Hardik Sharma",
       "Evan O'Toole",
       "Jaya Prakash Champati",
       "Kui Wu"
     ],
     "year": 2026,
     "venue": "arXiv preprint",
     "arxiv_id": "2601.22132",
     "doi": "10.48550/arXiv.2601.22132"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "LLM Shepherding requests only a short prefix (hint) from a large model to improve small model accuracy, achieving 42–94% cost reduction relative to LLM-only inference. Reactive Shepherding achieves the highest Accuracy-per-Cost Efficiency (ACE) across all four benchmarks (GSM8K, CNK12, HumanEval, MBPP), outperforming routing and cascading baselines. Even small hints (10–30% of the full LLM response) yield substantial accuracy gains, and the approach generalizes zero-shot from math to code generation tasks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All four benchmarks are publicly available: GSM8K (Cobbe et al., 2021), CNK12 from OpenR1-Math-220k (OpenR1, 2025), HumanEval (Chen, 2021), and MBPP (Austin et al., 2021). The paper uses these without modification (except sampling CNK12)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions NVIDIA RTX 5090 GPU and Groq API but provides no requirements.txt, Dockerfile, or detailed environment/dependency specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1–4 report only point estimates for accuracy and cost. No confidence intervals or error bars are provided despite running 7 independent trials per query."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims shepherding outperforms baselines based solely on comparing numbers in tables. No statistical significance tests (p-values, t-tests, bootstrap tests) are used to support these comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cost reductions are reported as percentages with baseline context (e.g., '42–94% relative to LLM-only inference'), and ACE ratios provide normalized efficiency comparisons (e.g., 2.78 for Reactive Shepherding on MBPP vs 1.76 for ABC). Absolute costs and accuracies are shown for all methods."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the sample sizes used. GSM8K test set is 776 (subset of 1,319), CNK12 samples 21,471 from a larger dataset, but neither choice is justified with a power analysis or rationale."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '7 independent trials per query and apply majority voting' (Section 4.1) but does not report standard deviation, variance, or any spread measure across these trials."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Four baselines are compared: two routing methods (RouteLLM, GraphRouter) and two cascading methods (FrugalGPT, ABC), plus SLM-only and LLM-only reference points."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are recent: RouteLLM (Ong et al., 2025), GraphRouter (Feng et al., 2025), FrugalGPT (Chen et al., 2024), and ABC (Kolawole et al., 2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (DeBERTa encoder, hint/no-hint classifier, hint size regressor, cascading decision module) but no systematic ablation removes individual components. Proactive vs reactive comparison and hint size sweeps (Figure 2) exist but are not framed as controlled single-variable ablations."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported using accuracy, total cost, cost reduction percentage, and the ACE metric (accuracy gain per unit cost)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates on math (exact numerical match) and code generation (unit test pass/fail) benchmarks where correctness is binary and deterministic. Human evaluation is irrelevant to these claims."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Explicit train/validation/test splits are described: GSM8K (776 test, 543 validation), CNK12 (2,147 test, 1,073 validation, 18,251 train). HumanEval and MBPP are used as zero-shot test sets with no dataset-specific training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per dataset (Tables 1–4), per strategy type (routing vs cascading vs shepherding), and per operating mode (proactive vs reactive). Figure 3 shows per-dataset cost comparisons under accuracy constraints."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses non-monotonic quality where excessive hints degrade performance (Figure 2, HumanEval), heavy-tailed hint distributions (Section 3, Appendix D), unsolvable queries that exceed SLM capabilities even with 90% hints (8.7% in CNK12), and proactive shepherding underperforming on some datasets."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: non-monotonic quality (hints can hurt on HumanEval at high sizes, Figure 2), proactive shepherding achieving lower accuracy than routing on CNK12 (77.9% vs 82.6%), and the framework being limited to verifiable tasks (Section 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims match results: '42–94% cost reduction' is supported by Tables 1–4 (67.4% GSM8K, 42.1% CNK12, 44.3% HumanEval, 93.6% MBPP). '2.8× cost reduction vs cascading' matches HumanEval comparison (44.3% vs 15.8%). 'Hints comprising 10–30% improve SLM accuracy' is supported by Figure 2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims ('shepherding reduces costs', 'hints improve accuracy') are supported by controlled comparisons: same datasets, same models, same evaluation protocol across all methods. The oracle cost analysis (Proposition 2.1, Corollary 2.2) provides theoretical justification. Ablation-like comparisons (proactive vs reactive, varying hint sizes) support component-level causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'LLM Shepherding for Cost-Efficient Inference' and conclusion 'a new approach for cost-efficient deployment' frame the contribution broadly. However, experiments use only one SLM-LLM pair (Llama 3.2/3.3 from the same family with shared tokenizer) on four benchmarks in two domains (math and code). Section 5 acknowledges some limitations but the core framing extends beyond the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for why shepherding works well. For example, the shared tokenizer and model family could be a major factor, but this is only briefly mentioned as a scope limitation in Section 5 rather than as an alternative explanation for the strong results."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy (exact match for math, unit test pass for code) and monetary cost ($), and frames claims in terms of these specific metrics. The ACE metric is explicitly defined. No gap exists between measurements and claims."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are provided: 'Llama-3.2-3B-Instruct' (SLM), 'Llama-3.3-70B-Versatile' (LLM via Groq API), and 'DeBERTa-v3-large' (predictor backbone). These are identifiable model versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Appendix E describes prompt structure with a template ('Instruction: Produce a final answer... Question: {query} Hint: {hint}') and mentions 'a fixed few-shot prompt consisting of two example question–answer pairs' for GSM8K, but the actual few-shot examples are not provided. The reader cannot reconstruct the exact prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Key hyperparameters are reported: temperature 0.3, top-p 0.95 for generation; K=2 or K=3 for reactive consensus; threshold values α and ηhint per dataset (Table 5, Appendix F); Groq API pricing ($0.59/$0.79 per 1M tokens); AdamW optimizer with EMA (Appendix B.2)."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The system is a standard predict-then-query pipeline without tools, retry logic, or feedback mechanisms."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Appendix B.1 detail the training data construction pipeline: discretized token budgets at 10% increments, deterministic LLM decoding for labeling, filtering criteria. Appendix C describes outlier filtering with distribution plots (Figures 4–5). Dataset splits are explicitly stated (Section 4.1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Scope and Future Directions' provides a dedicated discussion of limitations including restriction to verifiable tasks and same model family experiments."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 identifies study-specific threats: (1) the method is only validated on verifiable reasoning tasks where correctness is automatically assessable, (2) SLM-LLM pairs are from the same model family sharing a common tokenizer, and extending to different architectures is an open question."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states: 'This work focuses on verifiable reasoning tasks, including mathematical problem-solving and code generation.' It also identifies what was NOT tested: open-ended tasks, cross-family model pairs with different tokenizers."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw experimental data (model outputs, per-query results, training labels) is released for independent verification. Only aggregated results in tables are shown."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 and Appendix B.1 describe in detail how training supervision labels were constructed: discretized hint sizes at 10% increments, LLM calls with max_new_tokens, SLM evaluation at each hint level. Benchmark sources are cited with specific versions."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (GSM8K, CNK12, HumanEval, MBPP)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: benchmark sampling (Section 4.1), supervision label construction with 10 LLM calls per query (Appendix B.1), outlier filtering with distribution analysis and counts (Appendix C, Figures 4–5), train/validation/test splits with exact sizes."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information, acknowledgments section, or grant numbers are mentioned anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Victoria (Dong, O'Toole, Champati, Wu) and Manipal University (Sharma). They are not evaluating their own commercial product."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed, so independence cannot be assessed. The absence of a funding statement does not confirm the absence of funding."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff date for Llama-3.2-3B-Instruct or Llama-3.3-70B-Versatile. This is needed to assess whether benchmarks overlap with training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K (2021), HumanEval (2021), or MBPP (2021) problems appeared in the Llama models' training data. These benchmarks predate the models and could be contaminated."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K, HumanEval, and MBPP were all published in 2021, well before Llama 3's training. These benchmarks were widely available online and could be in the training data. This contamination risk is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is central to the paper. Tables 1–4 report per-strategy costs in dollars. API pricing is stated ($0.59/1M input, $0.79/1M output). Section 4.4 reports latency (7.32 ms policy overhead, 384.62 ms SLM generation per run)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While inference hardware is mentioned (NVIDIA RTX 5090) and API costs are reported, the total computational budget for training the DeBERTa-based predictor, generating supervision labels (10 LLM calls per training query), and running outlier analysis is not quantified."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper runs 7 independent trials per query with majority voting but does not report sensitivity across different random seeds or variance in final accuracy/cost metrics."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 states: 'we perform 7 independent trials per query and apply majority voting.' Reactive Shepherding uses K=2 or K=3 SLM responses for consensus (Section 3.5, Table 5)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Appendix B.2 mentions 'grid search on the validation set to calibrate the policy' but does not state the number of configurations tried, compute spent on the search, or the search grid dimensions."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B.2 describes selection via grid search on the validation set, and Table 5 shows the final configurations used per dataset. Configurations are selected to maximize accuracy under cost constraints on validation data, not test data."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'default parameter settings released by the authors in their respective GitHub repositories' for baselines (Section 4.2) but does not acknowledge that their own method is tuned while baselines use defaults, creating a potential systematic advantage."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Cost-performance tradeoffs are central: Tables 1–4 show cost and accuracy for all methods, Figure 3 shows minimum cost to meet accuracy targets, and the ACE metric explicitly normalizes accuracy gain per unit cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses GSM8K, CNK12, HumanEval, and MBPP without discussing whether these benchmarks validly measure the capabilities claimed (mathematical reasoning, code generation ability). No discussion of construct validity or benchmark limitations."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All methods use the same direct API/inference pipeline. Comparisons are between inference strategies, not between models in different scaffolds."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), HumanEval (2021), and MBPP (2021) were published years before the Llama 3.2/3.3 models were trained. The paper does not discuss whether these benchmark solutions appeared in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup could leak information. The hint mechanism provides LLM partial outputs to the SLM — while this is intentional, the paper does not analyze whether hints contain memorized benchmark answers rather than genuine reasoning."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The training data for the shepherding model is constructed from the same benchmarks used for evaluation (GSM8K train split for GSM8K test, CNK12 train split for CNK12 test). While train/test splits are used, structural similarity and potential distributional overlap are not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap, decontamination) is applied to any of the benchmarks."
       }
     }
   },
   "claims": [
     {
       "claim": "LLM Shepherding reduces costs by 42–94% relative to LLM-only inference across four benchmarks.",
       "evidence": "Tables 1–4 show Reactive Shepherding cost reductions: 67.4% (GSM8K), 42.1% (CNK12), 44.3% (HumanEval), 93.6% (MBPP).",
       "supported": "strong"
     },
     {
       "claim": "Reactive Shepherding achieves the highest ACE on all datasets, up to 2.78 on MBPP.",
       "evidence": "Tables 1–4 show ACE values: 1.97 (GSM8K), 1.25 (CNK12), 1.42 (HumanEval), 2.78 (MBPP). All are highest among compared methods.",
       "supported": "strong"
     },
     {
       "claim": "Even hints comprising 10–30% of the full LLM response improve SLM accuracy significantly.",
       "evidence": "Figure 2 shows SLM accuracy rising substantially with small hints across all four datasets, with diminishing returns beyond 60%.",
       "supported": "strong"
     },
     {
       "claim": "Shepherding achieves lower cost than routing and cascading under oracle decision-making.",
       "evidence": "Corollary 2.2 provides a mathematical proof that c*_shep(q) ≤ c*_route(q) = c*_casc(q) when SLM cost is zero. Oracle rows in Tables 1–4 confirm empirically.",
       "supported": "strong"
     },
     {
       "claim": "Shepherding transfers zero-shot from math to code generation, matching cascading accuracy on HumanEval at 2.8× cost reduction.",
       "evidence": "Table 3 shows Reactive Shepherding matches ABC accuracy (76.2%) with 44.3% cost reduction vs ABC's 15.8%, using a GSM8K-trained model without code-specific training (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "This is the first work to exploit token-level budget control for SLM-LLM collaboration.",
       "evidence": "Section G.5 positions the work against routing, cascading, speculative decoding, and knowledge distillation, arguing none use partial output control. No prior work on hint-based collaboration is cited.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Single model family tested",
       "detail": "All experiments use Llama 3.2-3B and Llama 3.3-70B, which share the same tokenizer. The paper acknowledges this in Section 5 but does not test any cross-family pairs. The approach's effectiveness may depend on shared tokenization, limiting generalizability."
     },
     {
       "flag": "No error bars despite multiple trials",
       "detail": "The paper runs 7 independent trials per query with majority voting but reports only final point estimates in Tables 1–4. Variance across these trials is never shown, making it impossible to assess result stability or whether differences between methods are statistically significant."
     },
     {
       "flag": "Benchmark contamination risk ignored",
       "detail": "GSM8K, HumanEval, and MBPP were all published in 2021, years before Llama 3 training. These benchmarks are widely reproduced online. If the LLM has memorized solutions, the 'hints' may simply be regurgitating memorized answers rather than demonstrating genuine reasoning assistance. This could inflate both baseline and shepherding accuracies."
     },
     {
       "flag": "Potentially unfair baseline comparison",
       "detail": "Section 4.2 states baselines use 'default parameter settings released by the authors in their respective GitHub repositories,' while Shepherding configurations are tuned per-dataset (Table 5). Baselines may be operating at suboptimal settings for these specific benchmarks."
     },
     {
       "flag": "No code released",
       "detail": "No source code, trained models, or reproduction artifacts are provided. The DeBERTa-based predictor, training pipeline, and prompt templates cannot be independently verified or reproduced."
     }
   ],
   "cited_papers": [
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu", "W.-L. Chiang", "T. Wu", "J. E. Gonzalez", "M. W. Kadous", "I. Stoica"],
       "year": 2025,
       "relevance": "Routing framework that learns to direct queries to different LLMs based on preference data, directly compared as a baseline."
     },
     {
       "title": "GraphRouter: A Graph-Based Router for LLM Selections",
       "authors": ["T. Feng", "Y. Shen", "J. You"],
       "year": 2025,
       "relevance": "Graph-based LLM routing method using heterogeneous graphs for task-query-LLM interactions, used as a routing baseline."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2024,
       "relevance": "Pioneering LLM cascading approach demonstrating up to 98% cost reduction through learned model combinations."
     },
     {
       "title": "Agreement-Based Cascading for Efficient Inference",
       "authors": ["S. Kolawole", "D. Dennis", "A. Talwalkar", "V. Smith"],
       "year": 2025,
       "relevance": "Training-free cascading strategy using response agreement for escalation decisions, used as a cascading baseline."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark for code generation evaluation, one of the four evaluation benchmarks used."
     },
     {
       "title": "Training Verifiers to Solve Math Word Problems",
       "authors": ["K. Cobbe", "V. Kosaraju", "M. Bavarian", "M. Chen", "H. Jun", "L. Kaiser", "M. Plappert", "J. Tworek", "J. Hilton", "R. Nakano", "C. Hesse", "J. Schulman"],
       "year": 2021,
       "arxiv_id": "2110.14168",
       "relevance": "Introduces GSM8K, a grade-school math benchmark used as one of the primary evaluation datasets."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["J. Austin", "A. Odena", "M. Nye", "M. Bosma", "H. Michalewski", "D. Dohan", "E. Jiang", "C. Cai", "M. Terry", "Q. Le", "C. Sutton"],
       "year": 2021,
       "relevance": "Introduces the MBPP benchmark for entry-level Python programming tasks, used as a code generation evaluation dataset."
     },
     {
       "title": "The Llama 3 Herd of Models",
       "authors": ["A. Dubey"],
       "year": 2024,
       "relevance": "Describes the Llama 3 model family used as both the SLM (3.2-3B) and LLM (3.3-70B) in all experiments."
     },
     {
       "title": "Automix: Automatically Mixing Language Models",
       "authors": ["P. Aggarwal", "A. Madaan", "A. Anand", "S. P. Potharaju", "S. Mishra", "P. Zhou", "A. Gupta", "D. Rajagopal", "K. Kappaganthu", "Y. Yang"],
       "year": 2024,
       "relevance": "LLM cascading approach using few-shot self-verification and POMDP-based routing for cost reduction."
     },
     {
       "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing",
       "authors": ["D. Ding", "A. Mallick", "C. Wang", "R. Sim", "S. Mukherjee", "V. Rühle", "L. V. S. Lakshmanan", "A. H. Awadallah"],
       "year": 2024,
       "relevance": "Quality-aware LLM query routing that dynamically trades quality for cost based on predicted query difficulty."
     },
     {
       "title": "Language Model Cascades: Token-Level Uncertainty and Beyond",
       "authors": ["N. Gupta", "H. Narasimhan", "W. Jitkrittum", "A. S. Rawat", "A. K. Menon", "S. Kumar"],
       "year": 2024,
       "relevance": "Token-level uncertainty approach for LLM cascading decisions, closely related to shepherding's token-level control."
     },
     {
       "title": "MiniLLM: Knowledge Distillation of Large Language Models",
       "authors": ["Y. Gu", "L. Dong", "F. Wei", "M. Huang"],
       "year": 2024,
       "relevance": "Knowledge distillation method for LLMs using reverse KL divergence, complementary approach to inference-time collaboration."
     },
     {
       "title": "Small Language Models (SLMs) Can Still Pack a Punch: A Survey",
       "authors": ["S. Subramanian", "V. Elango", "M. Gungor"],
       "year": 2025,
       "arxiv_id": "2501.05465",
       "relevance": "Survey of SLM capabilities relevant to the cost-quality tradeoff motivation of shepherding."
     }
   ]
 }