ai-research-survey

scan.json (30231B)

---

 {
   "paper": {
     "title": "Do We Truly Need So Many Samples? Multi-LLM Repeated Sampling Efficiently Scales Test-Time Compute",
     "authors": [
       "Jianhao Chen",
       "Zishuo Xun",
       "Bocheng Zhou",
       "Han Qi",
       "Hangfan Zhang",
       "Qiaosheng Zhang",
       "Yang Chen",
       "Wei Hu",
       "Yuzhong Qu",
       "Wanli Ouyang",
       "Shuyue Hu"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2504.00762",
     "doi": "10.48550/arXiv.2504.00762"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "ModelSwitch, a multi-LLM repeated-sampling-then-voting strategy that dynamically switches between models based on answer consistency, outperforms single-LLM self-consistency and multi-agent debate methods across seven benchmarks while reducing average sampling cost by 34%. On MMLU-Pro, ModelSwitch achieves 63.2% accuracy, surpassing the best single LLM by 10.2 points and all debate-based methods. The paper provides theoretical analysis showing that mixing models with complementary error distributions can yield correct answers even when neither model individually succeeds, with expected sampling reduction of 1/(n×c) where n is the number of models.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are available at https://github.com/JianhaoChen-nju/ModelSwitch' on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states code and data are available at the GitHub link. Additionally, all seven benchmarks used (GSM8K, MATH, MathBench, MGSM, DATE, MMLU-Pro, AIME24) are publicly available datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A.1 mentions the compute cluster (Ubuntu 22.04, 8 NVIDIA A100 GPUs, 80GB VRAM, 1600GB memory) but provides no requirements.txt, Dockerfile, or detailed library version specifications needed to recreate the environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README content, commands, or scripts for replicating experiments are described."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All main results are reported as point estimates (e.g., '81% accuracy', '63.2% accuracy'). No confidence intervals or error bars appear in any of the figures or tables for the primary performance claims."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims ModelSwitch 'outperforms' baselines based solely on comparing point accuracy numbers (e.g., Figure 4, Figure 5) without any statistical significance tests. Correlation analyses in Section 2 report r and p-values, but no significance tests are applied to the main comparative claims."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute accuracy differences with baselines (e.g., '10.2-point increase over best single LLM' on MMLU-Pro), percentage cost savings ('34% samples on average'), and efficiency ratios ('14× more efficient'). Baseline values are always provided for context."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for dataset sizes or for the choice of 500 randomly selected MMLU-Pro questions. The sampling budget of 16 is used as a convention but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. Results appear to be from single experimental runs with stochastic sampling (temperature=1) but no repeated-run statistics."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines: self-consistency for individual LLMs (GPT-4o mini, Gemini 1.5 Flash), stronger single models (GPT-4o, Gemini 1.5 Pro), and multi-agent methods (MAD, AgentVerse, ChatEval, MOA). Section 4.2-4.3."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include MAD (ICML 2024), AgentVerse (ICLR 2024), ChatEval (ICLR 2024), MOA (2024), and self-consistency (ICLR 2023). These represent recent state-of-the-art multi-agent and multi-LLM approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides an ablation study of the weighted voting algorithm, removing internal weights, external weights, and both. Section 4.4 ablates the number and order of LLMs."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both accuracy (efficacy) and average actual sampling count per query (efficiency), as well as API cost comparisons (Table 3). These capture different dimensions of performance."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation is included. All evaluation is automated via answer matching on benchmark datasets with known correct answers."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Standard test splits are used for all seven benchmarks (e.g., GSM8K test set of 1,319 questions, MATH 500, MathBench Arith 300). These are established test sets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down across all seven individual datasets (Figures 4, 5, 6, 7) and across different sampling budgets, providing per-dataset performance rather than only aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No specific failure case analysis is provided. The paper does not examine individual questions where ModelSwitch fails or analyze error patterns."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 reports that scaling from 2 to 6 models can hurt performance (78.6%→76.4% on DATE under strong-to-weak). The paper notes 'simply increasing the number of models does not continuously enhance performance.'"
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are supported: outperforms self-consistency (Figure 4), outperforms multi-agent debate (Figure 5), reduces inference costs (Table 3), requires only few comparable LLMs (Figure 6), can be extended with verification (Figure 7), and theoretical analysis is provided (Section 5)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper's main causal claims ('ModelSwitch improves performance') are supported by controlled ablation studies (Table 4 removes individual components) and Section 4.4 systematically varies the number and order of models. Theoretical analysis in Section 5 provides formal conditions. The ablation design adequately isolates component contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims applicability to 'various reasoning and knowledge-based tasks' (Conclusion) and the title implies broad test-time compute scaling. However, 5 of 7 benchmarks are math-focused (GSM8K, MATH, MathBench, MGSM, AIME24) with only DATE (symbolic reasoning) and MMLU-Pro (multitask) outside mathematics. The generalization to non-mathematical domains is not bounded."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider confounds such as whether the improvements are specific to the particular model pairs chosen, whether the benchmarks have properties that particularly favor multi-model voting, or whether the consistency signal is merely a proxy for question difficulty."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims match its measurements: it measures accuracy on benchmarks and frames it as accuracy; it measures sampling counts and frames it as efficiency. No proxy gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names: 'GPT-4o mini', 'Gemini 1.5 Flash', 'Claude 3 Haiku', without snapshot dates or API versions. Open-source models are specified more precisely (e.g., 'Llama-3.1-8B-Instruct', 'Qwen2.5-7B-Instruct') but closed-source models lack version specificity."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'All our queries are asked in COT format by default' (Section 4.2) but does not provide the actual prompt text or chain-of-thought prompt templates used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only GPT-4o mini's hyperparameters are specified: 'temperature and top_p of GPT-4o mini to 1' (Appendix A.1). All other LLMs 'were kept with their default hyperparameters' without stating what those defaults are."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The method is a sampling and voting algorithm without agent loops, tool use, or scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes each dataset with sample counts and subsets used (e.g., MathBench Arith subset with 300 questions, MGSM 10 non-English languages with 1,000 samples, MMLU-Pro random 500-question subset)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists in the paper. The conclusion mentions the approach is 'practical and generalizable' without discussing any limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not describe what the results do NOT show, what settings or tasks are excluded, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Code and data are available at https://github.com/JianhaoChen-nju/ModelSwitch.' All benchmarks used are also publicly available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is straightforward: standard benchmarks are used. Appendix A.1 describes each benchmark, its size, and how subsets were selected (e.g., random 500 from MMLU-Pro, 10 non-English languages from MGSM)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data consists of standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Algorithm 1 describes the full ModelSwitch pipeline. The experimental setup (Section 4.1) describes how queries are processed, how answers are extracted, and how voting is performed."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding or acknowledgments section is present in the paper. Authors are from multiple institutions (Nanjing University, Shanghai AI Lab, University of Auckland, Penn State) but no funding sources are disclosed."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed on the first page. Authors are from academic institutions and Shanghai AI Laboratory. They evaluate third-party models (GPT-4o, Gemini, Claude) rather than their own products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making it impossible to assess funder independence. Shanghai AI Laboratory affiliations could represent institutional interest in showing multi-LLM approaches work."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models used (GPT-4o mini, Gemini 1.5 Flash, Claude 3 Haiku, or any open-source models)."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of potential train/test overlap. GSM8K, MATH, and other benchmarks have been publicly available for years and could be in the training data of models used."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Several benchmarks (GSM8K published 2021, MATH 2021) were available before the training of models like GPT-4o mini and Gemini 1.5 Flash. No contamination analysis or discussion is provided."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All experiments are automated benchmark evaluations."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports actual API costs in dollars for each dataset and method. The paper also reports cost reduction percentages (15%-48% across datasets) and average sampling count reductions (34% on average)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 describes compute resources (Ubuntu 22.04, 8 NVIDIA A100 GPUs, 80GB VRAM each, 1600GB memory). Table 3 provides total API costs. Minimum hardware requirements for different model sizes are also stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. Results are reported as single runs with stochastic sampling (temperature=1) but no repeated-seed evaluation."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. The sampling budget K is stated per query, but whether the entire experiment was repeated multiple times is not specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "External weights Wβ (Tables 1-2) appear tuned per dataset but no search budget, search method, or number of configurations tried is reported."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The Wβ values in Tables 1 and 2 are presented without justification for how they were selected. No validation set selection procedure or search methodology is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparisons across 7 datasets and 6+ methods without any correction for multiple comparisons or even basic significance testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement ModelSwitch and compare against other methods without acknowledging the bias of evaluating their own system. No independent evaluation or discussion of this bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figures 4 and 7 explicitly show performance as a function of sampling budget (1 to 16). Table 3 compares costs at matched budgets. The comparison is always at matched total sampling budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the benchmarks actually measure the claimed capabilities. The paper uses GSM8K, MATH, etc. without questioning whether accuracy on these benchmarks reflects the broader 'reasoning and knowledge-based' capabilities claimed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. The method is a sampling and voting strategy applied directly to model outputs."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. Several benchmarks (GSM8K, MATH) were published years before the models' training periods, creating potential temporal leakage."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. The evaluation uses direct question-answering so feature leakage risk is lower, but the paper does not discuss this."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Not addressed. No discussion of whether training data for the evaluated models contains the benchmark questions or structurally similar problems."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference tests, or decontamination pipelines."
       }
     }
   },
   "claims": [
     {
       "claim": "ModelSwitch outperforms single-LLM self-consistency across all seven benchmarks while reducing average sampling count by 34%.",
       "evidence": "Figure 4 shows ModelSwitch consistently outperforming self-consistency for both GPT-4o mini and Gemini 1.5 Flash across 6 datasets. Average actual sampling counts are 9.2-13.4 out of budget 16, saving 34% on average (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "ModelSwitch achieves 63.2% accuracy on MMLU-Pro, surpassing the best single LLM by 10.2 points and all multi-agent debate methods.",
       "evidence": "Figure 5 shows ModelSwitch at 63.2% vs. best single LLM at 53%, MAD at 47.6%, AgentVerse at 44.8%, ChatEval at 44%, MOA at 52.6% (Section 4.3).",
       "supported": "moderate"
     },
     {
       "claim": "Only a few comparable LLMs are needed to achieve optimal ModelSwitch performance; adding more models does not continuously improve results.",
       "evidence": "Figure 6 shows that scaling from 1→2 models provides the biggest gain, while 2→6 models shows plateau or decline (e.g., 78.6%→76.4% on DATE). Section 4.4.",
       "supported": "moderate"
     },
     {
       "claim": "There is a universal correlation between consistency (entropy) and accuracy across LLMs and datasets.",
       "evidence": "Figure 2 shows moderate to high correlations (r = 0.61 to 0.96, all p < 0.001) between entropy and accuracy for 6 LLMs on MATH and MathBench with 16 samples per query (Section 2).",
       "supported": "strong"
     },
     {
       "claim": "ModelSwitch can be combined with reward models (RM-BoN) for further performance improvement.",
       "evidence": "Figure 7 shows ModelSwitch + RM-BoN achieving 84% on both MATH and MathBench with 16 samples, vs. 82% and 81.33% for best single LLM + RM-BoN (Section 4.5).",
       "supported": "moderate"
     },
     {
       "claim": "The expected number of samplings can be reduced by a factor 1/(n×c) where n is the number of models and c is the consistent-answer probability.",
       "evidence": "Theoretical bound derived in Section 5, Equation 3. The bound holds under the assumption that each model's probability of producing a consistent answer list exceeds constant c > 0.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "All main results (Figures 4, 5, 6, 7) report single point estimates with no confidence intervals, error bars, or variance across runs. Given stochastic sampling (temperature=1), results could vary substantially across repetitions."
     },
     {
       "flag": "Heavy math benchmark bias",
       "detail": "5 of 7 benchmarks (GSM8K, MATH, MathBench, MGSM, AIME24) are mathematics-focused. Only DATE (symbolic reasoning) and MMLU-Pro (multitask) test non-mathematical domains. Claims of generality to 'various reasoning and knowledge-based tasks' are not well-supported."
     },
     {
       "flag": "Unexplained hyperparameter tuning",
       "detail": "External weights Wβ (Tables 1-2) vary across datasets and models (values of 1, 1.5, or 2) with no explanation of how they were selected, whether a validation set was used, or how sensitive results are to these choices."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Several benchmarks (GSM8K, MATH published 2021) have been publicly available for years and may be in the training data of GPT-4o mini, Gemini 1.5 Flash, and other models. No contamination analysis is performed."
     },
     {
       "flag": "Missing limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or scope boundaries section. No discussion of when or why ModelSwitch might fail, or what settings it is not applicable to."
     },
     {
       "flag": "Claims rated moderate despite impressive numbers",
       "detail": "The MMLU-Pro result (63.2% vs. 53% best single LLM) is a 10.2pp improvement but reported without statistical tests or repeated runs. With stochastic sampling and only 500 randomly-selected questions, the true difference could be smaller."
     }
   ],
   "cited_papers": [
     {
       "title": "Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters",
       "authors": ["Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar"],
       "year": 2024,
       "arxiv_id": "2408.03314",
       "relevance": "Core reference on test-time compute scaling as an alternative to parameter scaling, which ModelSwitch builds upon."
     },
     {
       "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling",
       "authors": ["Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich"],
       "year": 2024,
       "arxiv_id": "2407.21787",
       "relevance": "Demonstrates that scaling repeated sampling consistently improves coverage of correct answers, motivating the sample efficiency problem this paper addresses."
     },
     {
       "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
       "authors": ["Xuezhi Wang", "Jason Wei", "Dale Schuurmans"],
       "year": 2023,
       "relevance": "The foundational self-consistency method that ModelSwitch extends to the multi-LLM setting."
     },
     {
       "title": "Improving Factuality and Reasoning in Language Models through Multiagent Debate",
       "authors": ["Yilun Du", "Shuang Li", "Antonio Torralba"],
       "year": 2024,
       "relevance": "Multi-agent debate baseline (MAD) that ModelSwitch outperforms on multiple benchmarks."
     },
     {
       "title": "AgentVerse: Facilitating Multi-Agent Collaboration and Exploring Emergent Behaviors",
       "authors": ["Weize Chen", "Yusheng Su", "Jingwei Zuo"],
       "year": 2024,
       "relevance": "Multi-agent collaboration framework used as a baseline, outperformed by ModelSwitch."
     },
     {
       "title": "ChatEval: Towards Better LLM-Based Evaluators through Multi-Agent Debate",
       "authors": ["Chi-Min Chan", "Weize Chen", "Yusheng Su"],
       "year": 2024,
       "relevance": "Multi-agent debate evaluation system used as a baseline in the multi-agent comparison."
     },
     {
       "title": "Mixture-of-Agents Enhances Large Language Model Capabilities",
       "authors": ["Junlin Wang", "Jue Wang", "Ben Athiwaratkun"],
       "year": 2024,
       "arxiv_id": "2406.04692",
       "relevance": "MOA architecture that aggregates samples from multiple LLMs, serving as the strongest multi-LLM baseline."
     },
     {
       "title": "If Multi-Agent Debate is the Answer, What is the Question?",
       "authors": ["Hangfan Zhang", "Zhiyao Cui", "Xinrun Wang"],
       "year": 2025,
       "arxiv_id": "2502.08788",
       "relevance": "Questions when multi-agent debate helps, finding that different LLMs excel at different questions—supporting ModelSwitch's complementary strengths hypothesis."
     },
     {
       "title": "Can 1B LLM Surpass 405B LLM? Rethinking Compute-Optimal Test-Time Scaling",
       "authors": ["Runze Liu", "Junqi Gao", "Jian Zhao"],
       "year": 2025,
       "arxiv_id": "2502.06703",
       "relevance": "Explores test-time compute scaling to bridge model size gaps, directly related to this paper's claim of achieving large-model performance with smaller models."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang", "Haowei Zhang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Reasoning-focused LLM used in the extended experiments with reasoning models (Appendix A.2)."
     },
     {
       "title": "RouteLLM: Learning to Route LLMs from Preference Data",
       "authors": ["Isaac Ong", "Amjad Almahairi", "Vincent Wu"],
       "year": 2024,
       "relevance": "Model routing approach that trains routers to distribute queries to specialized models, conceptually related to ModelSwitch's training-free routing via consistency."
     },
     {
       "title": "More Agents Is All You Need",
       "authors": ["Junyou Li", "Qin Zhang", "Yangbin Yu"],
       "year": 2024,
       "arxiv_id": "2402.05120",
       "relevance": "Demonstrates that scaling the number of sampling agents improves performance, directly related to the repeated sampling paradigm."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The method is simple to implement (multi-API sampling + voting) and reduces API costs by 34%, making it immediately applicable to practitioners using multiple LLM providers."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "The finding that two weak models can outperform strong models via simple mixing is mildly surprising but the complementary strengths intuition is not deeply contrarian."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications discussed or relevant."
     },
     "drama_conflict": {
       "score": 0,
       "justification": "No controversy or conflict. Standard benchmark comparison paper."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code is released on GitHub. The method is simple enough to implement independently, though it requires multiple API accounts."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "Shanghai AI Laboratory has moderate recognition. The paper uses well-known models (GPT-4o, Gemini, Claude) but is not from those labs."
     }
   }
 }