ai-research-survey

scan.json (27801B)

---

 {
   "paper": {
     "title": "Don't Always Pick the Highest-Performing Model: An Information Theoretic View of LLM Ensemble Selection",
     "authors": [
       "Yigit Turkmen",
       "Baturalp Buyukates",
       "Melih Bastopcu"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.08003"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. Algorithms are described in pseudocode (Algorithms 1-6) but no executable code is released."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available datasets: MEDMCQA (Pal et al., 2022), MMLU (Hendrycks et al., 2021), and IMDB movie reviews (Maas et al., 2011). All are standard public benchmarks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or library version information is provided. The paper does not specify software dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No README, reproduction scripts, or step-by-step instructions are provided. While pseudocode algorithms are given, there are no runnable instructions for replicating experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 4-6, 8-10, 12-14 report mean ± std dev across 30 evaluations. Figures include shaded standard deviation regions (e.g., 'Shaded region represents the standard deviation')."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'consistently outperforms strong baselines' but uses no statistical significance tests (no p-values, t-tests, bootstrap tests, or any other test). Improvements are compared by raw numbers only."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Error probabilities are reported with baseline context, e.g., 'achieving its best performance of 16.3% error at k=5, compared to 17.0% for Top-k selection' (Section 6.1). Tables provide absolute values enabling comparison."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for dataset sizes: 4183 MEDMCQA samples, 5000 randomly selected MMLU samples, 10000 IMDB samples. No power analysis or discussion of whether these sizes are adequate for the claimed effect sizes."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviation is reported across 30 evaluations (3 temperatures × 2 runs × 5 random 80/20 splits) in all summary tables. E.g., Table 4: '0.163 ± 0.008'."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are compared: Top-k (accuracy-based selection), Term 1 Only (relevance only), and Terms 1+2 (mRMR-style). Additionally, three aggregation rules are compared (MAP, MV, W-MV)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines represent the principal selection strategies in the literature. The mRMR baseline derives from Peng et al. (2005), the dominant paradigm. LLM-TOPLA (Tekin et al., 2024) and MUSE (Kruse et al., 2025) are discussed as related work."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The decomposition into Term 1 (accuracy), Terms 1+2 (mRMR), and full Greedy MI (all three terms) constitutes an ablation, testing the contribution of each component of the information gain decomposition (Theorem 4.3)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": false,
         "justification": "Only test error probability is reported. No other metrics (e.g., computational efficiency of selection, calibration, coverage, or area under budget-performance curve) are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is irrelevant to this paper's claims about ensemble selection algorithms for binary classification tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses an 80/20 random split: '80% for estimating the required information-theoretic quantities... 20% for evaluation' (Section 6.1). This is repeated across 5 independent random splits."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by dataset (MEDMCQA, MMLU, IMDB), by temperature setting (0.01, 0.3, 0.7), by run, and by ensemble size k. Per-condition curves appear in Appendices F-H."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.3 discusses IMDB where 'improvements from ensemble selection are modest' due to high correlation (ρ̄=0.90). Performance saturation at large k is discussed. Section 8 acknowledges binary setting limitation."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results: IMDB shows minimal gains; at large k all methods degrade; Appendix F.2 notes 'baselines with WE aggregation occasionally outperform Greedy MI with MAP for large ensemble sizes'; the saturation theorem itself is a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims 'consistently outperforms strong baselines under the same query budget' — results in Figures 5, 6 and Tables 4, 8, 12 show this for MEDMCQA and MMLU in mid-range k. IMDB gains are modest but present. The saturation floor claim is supported by Theorem 4.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims ('correlation introduces additional structure,' 'mRMR's aggressive diversity-seeking has forced it to include several weak models, degrading overall performance'). These are justified through formal theorems (4.1, 4.3, 4.4) and the ablation structure (Terms 1 vs 1+2 vs full)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 explicitly bounds scope: 'Our study focuses on a binary decision setting' and acknowledges 'extending these insights to richer output spaces and alternative dependency structures presents a promising direction.' The title and abstract don't overclaim beyond what's shown."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses alternative explanations: MAP estimation difficulty at large k, Gaussian-copula model limitations, the role of training dataset size for MI estimation. Appendix F.2 discusses aggregation rule interaction effects."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures test error probability and claims this measures ensemble classification accuracy. The measurement matches the claim directly — no proxy gap exists. The binary conversion procedure is transparent."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names only: 'GPT-4.1', 'GPT-5-chat', 'Gemini-2.5-flash', 'Claude-3.5-haiku', etc. No API versions, snapshot dates, or specific model IDs are provided. These are insufficient per the criterion."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendices F, G, H with exact text. E.g., MEDMCQA: 'System: Is the following statement true or false? Answer with a single word: True or False.' With concrete examples including fill values."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings explicitly stated: 0.01, 0.3, and 0.7 (Section 6.1). Laplace smoothing α=1 documented in Algorithm 2 (Appendix E.1). Two independent runs per temperature setting."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper sends single prompts to LLM APIs and collects binary predictions."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Binary conversion procedure is documented in detail (Section 6.1): positive queries append ground-truth answer, negative queries append a randomly selected incorrect answer. MMLU uses 5000 randomly selected samples from 14042. IMDB sampling described in Appendix H."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Limitations and Discussion' provides substantive discussion of the binary setting limitation, Gaussian-copula model assumptions, and saturation effects."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 identifies specific threats: binary decision setting may not extend to richer output spaces, Gaussian-copula may not capture all dependency structures, and saturation effects limit improvements. These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 states: 'Our study focuses on a binary decision setting, which allows for a clean and interpretable information-theoretic analysis and serves as a foundational step toward more general formulations.' Explicitly scopes to binary classification."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw model outputs (binary predictions from 12-13 LLMs on thousands of examples) are not released. Only aggregated results are shown. Independent verification of the correlation matrices and error patterns is not possible."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Data collection is described: models queried through OpenRouter (Appendix F), specific datasets and splits identified, temperature settings specified, binary conversion procedure documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks (MEDMCQA, MMLU, IMDB)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Pipeline documented: original multi-choice questions → binary conversion (2 queries per question) → LLM inference at specified temperatures → 80/20 random splits × 5 folds → MI estimation on train → evaluation on test."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding disclosed on page 1: 'This work was supported by Tubitak 2232-B program (Project No:124C533).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations clearly listed: Bilkent University (Turkmen, Bastopcu) and University of Birmingham (Buyukates). No affiliation with any evaluated model provider."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Tubitak is the Scientific and Technological Research Council of Turkey, a government research funding agency with no financial interest in which ensemble selection method performs best."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 12-13 LLMs evaluated. MEDMCQA (2022) and MMLU (2021) are old benchmarks likely in most models' training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MEDMCQA or MMLU examples appeared in the training data of GPT-4.1, GPT-5-chat, Qwen3-235b, or other evaluated models. This could affect the error correlation structure central to the paper's claims."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MEDMCQA (2022) and MMLU (2021) were published well before the training of the 2025-2026 models used. No contamination analysis is performed despite this being a significant concern for the validity of the observed correlation structures."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency figures reported. The paper queries 12-13 LLMs via OpenRouter across thousands of examples at three temperatures with two runs each, but total cost is not stated."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget (API spend, wall-clock time, or hardware) is reported despite extensive experimentation across multiple models, temperatures, and datasets."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across 2 independent runs per temperature and 5 random data splits. Standard deviations capture variability. Tables report 'averaged over 30 evaluations (3 temperatures × 2 runs × 5 folds).'"
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "Explicitly stated: '3 temperature settings (0.01, 0.3, and 0.7), performing two independent runs at each temperature' (Section 6.1) and '5 independent random splits' per run."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "The Greedy MI method is essentially parameter-free (greedy selection with standard Laplace smoothing α=1). Temperatures are experimental conditions, not hyperparameters of the selection method."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across all configurations (all k values, temperatures, runs, splits). No cherry-picking of best configuration — Figures 5-6 show full curves, Tables 4-14 show all settings."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Four methods are compared across up to 13 ensemble sizes, 3 temperatures, 2 runs, and 3 aggregation rules. No multiple comparison correction (Bonferroni, Holm, etc.) is applied."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement all baselines (Top-k, Term 1, mRMR) themselves and compare against their own method. No acknowledgment of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "The entire paper is structured around comparing methods at matched query budgets (k). Figures 5, 6 plot performance as a function of ensemble size k, which directly corresponds to compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether MEDMCQA, MMLU, and IMDB actually measure the capabilities relevant to ensemble selection claims. The artificial binary conversion (appending answers and asking true/false) creates a different task than the original benchmarks — this validity concern is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All models receive the same simple prompt with no agentic components."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MEDMCQA (2022) and MMLU (2021) were published years before the 2025-2026 models used. Models very likely trained on these benchmarks. No temporal leakage analysis is performed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The binary conversion format ('Is the answer X?') could systematically differ from how models encountered these questions in training. This potential feature leakage through format transformation is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Each multi-choice question generates one positive and one negative query that are structurally dependent (same question stem, correct vs incorrect answer). This non-independence in the evaluation set is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used (no canary strings, membership inference, temporal splits, or decontamination)."
       }
     }
   },
   "claims": [
     {
       "claim": "When LLM decisions are independent, the optimal ensemble selection is trivially selecting the most accurate models (Top-k).",
       "evidence": "Theorem 4.1 with formal proof in Appendix A, using binary symmetric channel (BSC) framework and stochastic degradation argument.",
       "supported": "strong"
     },
     {
       "claim": "Under uniform pairwise correlation, the error probability converges to a non-zero floor even with infinite models, given by Φ(Φ⁻¹(1−α)/√ρ).",
       "evidence": "Theorem 4.4 with formal proof in Appendix C using Gaussian-copula one-factor model and strong law of large numbers.",
       "supported": "strong"
     },
     {
       "claim": "The marginal information gain decomposes into accuracy, prediction redundancy, error correlation, and a label-dependence correction term, showing mRMR misses the error correlation component.",
       "evidence": "Theorem 4.3 with formal proof in Appendix B, demonstrating that standard mRMR criteria omit the I(Ej; ES) term.",
       "supported": "strong"
     },
     {
       "claim": "Greedy MI consistently outperforms strong baselines under identical query budgets across multiple datasets.",
       "evidence": "Figures 5-6, Tables 4, 8, 12. MEDMCQA: 16.3% vs 17.0% at k=5; MMLU: 14.1% vs 14.9% at k=6. Averaged over 30 evaluations with std dev.",
       "supported": "moderate"
     },
     {
       "claim": "The Gaussian-copula model effectively captures both pairwise and higher-order error dependencies of real LLM ensembles.",
       "evidence": "Figures 4, 10-11, 15-16, 20-21 show visual agreement between copula-predicted and empirical error distributions. Validation across all temperature/run settings.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["theoretical", "benchmark-eval"],
   "key_findings": "The paper provides an information-theoretic framework for LLM ensemble selection, proving that Top-k accuracy selection is optimal only under independent errors (Theorem 4.1) and that correlated errors create an irreducible error floor (Theorem 4.4). The Greedy MI algorithm, motivated by a novel accuracy-redundancy-error decomposition (Theorem 4.3), achieves modest but consistent improvements over baselines in the practical mid-budget range (k=3-7) on MEDMCQA (~0.7pp) and MMLU (~0.8pp), with negligible gains on IMDB where correlations are very high (ρ̄=0.90).",
   "red_flags": [
     {
       "flag": "No significance testing",
       "detail": "Claims of 'consistently outperforms' are based on raw number comparisons. Improvements are small (0.7-1.0 percentage points) and within the reported standard deviations. No statistical significance tests are performed to verify these differences are meaningful."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "MEDMCQA (2022) and MMLU (2021) are likely in the training data of 2025-2026 models (GPT-5-chat, GPT-4.1, Qwen3-235b, etc.). Contamination could systematically alter the error correlation structure that is central to the paper's theoretical and empirical claims."
     },
     {
       "flag": "Artificial binary conversion",
       "detail": "Multi-choice questions are converted to binary true/false by appending candidate answers. This creates non-independent positive/negative query pairs from the same question and transforms the task into something different from what the benchmarks were designed to measure. The validity of this conversion is not assessed."
     },
     {
       "flag": "Modest effect sizes",
       "detail": "Improvements over Top-k are ~0.7pp on MEDMCQA and ~0.8pp on MMLU. On IMDB, gains are negligible. These improvements, while consistent in direction, are small relative to the standard deviations reported and may not be practically significant."
     }
   ],
   "cited_papers": [
     {
       "title": "Why do multi-agent LLM systems fail?",
       "authors": ["Cemri, M.", "Pan, M. Z.", "Yang, S."],
       "year": 2025,
       "arxiv_id": "2503.13657",
       "relevance": "Systematic failure analysis of multi-agent LLM systems, identifies inter-agent misalignment as dominant failure mode — directly relevant to understanding correlated LLM errors."
     },
     {
       "title": "Towards a science of scaling agent systems",
       "authors": ["Kim, Y.", "Gu, K.", "Park, C."],
       "year": 2025,
       "arxiv_id": "2512.08296",
       "relevance": "Studies scaling laws for multi-agent systems, observes diminishing returns from coordination when single-agent accuracy exceeds ~45% — supports the saturation phenomenon studied here."
     },
     {
       "title": "Harnessing multiple large language models: A survey on LLM ensemble",
       "authors": ["Chen, Z.", "Li, J.", "Chen, P."],
       "year": 2025,
       "arxiv_id": "2502.18036",
       "relevance": "Survey of LLM ensemble methods covering voting, fusion, and selection strategies — comprehensive overview of the field this paper contributes to."
     },
     {
       "title": "Self-consistency improves chain of thought reasoning in language models",
       "authors": ["Wang, X.", "Wei, J.", "Schuurmans, D."],
       "year": 2023,
       "arxiv_id": "2203.11171",
       "relevance": "Foundational work on self-consistency through majority voting for single-model sampling, extended to multi-model settings by subsequent work."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["Chen, L.", "Zaharia, M.", "Zou, J."],
       "year": 2024,
       "relevance": "Addresses cost-effective LLM usage through cascaded selection and model routing — related to the budgeted ensemble selection problem."
     },
     {
       "title": "LLM-Blender: ensembling large language models with pairwise comparison and generative fusion",
       "authors": ["Jiang, D.", "Ren, X.", "Lin, B. Y."],
       "year": 2023,
       "relevance": "LLM ensemble method using pairwise ranking and generative fusion for combining outputs from multiple LLMs."
     },
     {
       "title": "Beyond majority voting: LLM aggregation by leveraging higher-order information",
       "authors": ["Ai, R.", "Pan, Y.", "Simchi-Levi, D."],
       "year": 2025,
       "arxiv_id": "2510.01499",
       "relevance": "Uses second-order statistics for optimal weight aggregation in LLM ensembles — complementary approach to the selection problem addressed here."
     },
     {
       "title": "LLM-TOPLA: Efficient LLM ensemble by maximising diversity",
       "authors": ["Tekin, S. F.", "Ilhan, F.", "Huang, T."],
       "year": 2024,
       "relevance": "Introduces focal diversity metrics for LLM ensemble pruning — alternative diversity-based approach to ensemble selection."
     },
     {
       "title": "Simple yet effective: An information-theoretic approach to multi-LLM uncertainty quantification",
       "authors": ["Kruse, M.", "Afshar, M.", "Khatwani, S."],
       "year": 2025,
       "relevance": "Applies Jensen-Shannon divergence for selecting well-calibrated LLM subsets — information-theoretic approach to LLM ensemble uncertainty."
     },
     {
       "title": "Cost-aware LLM-based online dataset annotation",
       "authors": ["Elumar, E. C.", "Tekin, C.", "Yagan, O."],
       "year": 2025,
       "relevance": "Uses Gaussian-copula sampler to model dependent correctness for Monte Carlo confidence estimation in LLM annotation — directly related copula modeling approach."
     },
     {
       "title": "Large language models: A survey",
       "authors": ["Minaee, S.", "Mikolov, T.", "Nikzad, N."],
       "year": 2025,
       "arxiv_id": "2402.06196",
       "relevance": "Comprehensive survey of LLM capabilities and applications, providing context for why LLM ensembling is needed."
     }
   ]
 }