ai-research-survey

scan.json (26760B)

---

 {
   "paper": {
     "title": "Pyramid MoA: A Probabilistic Framework for Cost-Optimized Anytime Inference",
     "authors": ["Arindam Khaled"],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.19509"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Pyramid MoA uses a lightweight classifier (Random Forest or XGBoost) trained on ensemble agreement features from small models (Llama-3-8B, Mistral-7B, Qwen-2.5-7B) to decide whether to escalate queries to a larger Oracle model (Llama-3-70B). On GSM8K, the system reports 91.4% accuracy at T=0.70 while escalating only 25.5% of queries (vs 79.1% for small models alone). On MBPP, ensemble agreement features outperformed self-reported model confidence for detecting buggy outputs (82.6% recall). The paper claims 93.0% accuracy with 61% cost reduction in the abstract, but only the 91.4%/T=0.70 operating point is detailed in the results section.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available standard benchmarks: GSM8K, MBPP, and HumanEval. These are public datasets that do not require the authors to release them."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, conda environment, or environment setup section is provided. Only model names are mentioned without dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions, README, or runnable scripts are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (93.0% accuracy, 82.6% recall, 91.4% accuracy) with no confidence intervals, error bars, or uncertainty quantification."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims Pyramid MoA outperforms baselines (FrugalGPT, Layer 1 alone) based solely on comparing point estimates. No statistical significance tests (p-values, t-tests, bootstrap tests, etc.) are reported."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: 91.4% vs 79.1% baseline on GSM8K, 93.0% vs 98.0% Oracle, 3.5x cost reduction, 40% lower inference cost than FrugalGPT on HumanEval. These provide sufficient context to understand effect magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The GSM8K test set size (N=1,052) is stated but not justified. The MBPP evaluation has only 23 buggy samples in the confusion matrix with no justification for why this is sufficient for the claims."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance across seeds, or multiple-run results are reported. All figures appear to be single-run point estimates."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against: Layer 1 only (79.1% baseline), Oracle-only (Llama-3-70B at 98.0%), and FrugalGPT cascade (Llama-8B → Llama-70B) on HumanEval (Section 3.2.2, Figure 3)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "FrugalGPT (2023) is a recent cost-efficiency baseline. The models used (Llama-3, Mistral-7B, Qwen-2.5-7B) are contemporary. Related work discusses recent MoA variants (Sparse MoA, Residual MoA, Self-MoA from 2024-2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "The system has multiple components (3-model ensemble, router, Oracle layer) but no systematic ablation removing individual components (e.g., testing with 2 vs 3 ensemble models, or removing the router). Feature importance (Figure 2b) shows variable importance within the router but is not a component ablation. Table 1 compares two router algorithms but is a configuration comparison, not ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports accuracy, recall (82.6% for bug detection), cost reduction (61%, 3.5x), and latency overhead (+0.82s) across experiments."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of system outputs is included. All evaluation is automated (benchmark accuracy, confusion matrices). Human evaluation of code generation quality beyond pass/fail could have been informative."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3.1 explicitly states: 'We evaluated the router on a strictly held-out test set (N = 1,052).' MBPP uses the standard test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "No per-category or per-difficulty breakdowns are provided. GSM8K results are a single aggregate accuracy. No analysis of performance on easy vs. hard problems, which is directly relevant to a routing system that claims to identify 'hard' queries."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No qualitative failure examples, error analysis, or discussion of when the router makes wrong escalation decisions (false positives/negatives beyond the confusion matrix counts)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 reports that Random Forest 'lacked the calibration precision required for the continuous, non-linear feature space of mathematical reasoning,' necessitating the switch to XGBoost. This is an explicit negative result about a configuration that did not work."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": false,
         "justification": "The abstract claims '93.0% accuracy, effectively matching the Oracle baseline (98.0%) while reducing compute costs by 61%.' However, Section 3.3.1 reports 91.4% accuracy at T=0.70 with 25.5% escalation (74.5% short-circuited). The 93.0%/61% operating point is asserted in the abstract and introduction but never shown in the detailed results. Additionally, 93.0% vs 98.0% is a 5pp gap — calling this 'effectively matching' is a stretch."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that ensemble agreement enables better routing than self-reported confidence — is supported by the feature importance analysis (Figure 2b) showing semantic agreement and output variance outperform self-confidence. The system-level claims (routing reduces cost while maintaining accuracy) are supported by the baseline comparisons."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Cost-Optimized Anytime Inference' broadly, and the Discussion says the framework 'democratizes access to high-end reasoning capabilities.' The paper tests on only GSM8K (math) and MBPP/HumanEval (code) with specific open-weight models. The broad framing exceeds the tested settings."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. For example: Could the cost savings be achieved by a simpler confidence threshold on a single model? Could the router's effectiveness be an artifact of the specific benchmark difficulty distribution? No confounds or robustness checks are considered."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures accuracy on benchmarks and compute cost, and claims cost-optimized inference. The measurements directly correspond to the claims — no proxy gap exists between what is measured and what is claimed at the empirical level."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'Llama-3-8B', 'Mistral-7B', 'Qwen-2.5-7B', 'Llama-3-70B' without specifying instruct vs. base variants, exact version numbers (e.g., Mistral-7B-v0.1 vs v0.3), or snapshot dates. These are model family names, not exact versions."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No actual prompt text is provided for how queries are sent to the ensemble models or the Oracle. The paper describes the architecture but never shows the prompts used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The routing threshold T=0.70 is reported, but LLM inference parameters (temperature, top-p, max tokens) are not mentioned. XGBoost and Random Forest hyperparameters are also not reported."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The system is a model routing/ensemble architecture, not agentic scaffolding. There is no tool use, iterative loops, memory management, or agent-like behavior."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how the router's training data was prepared, how features were extracted from model outputs, how train/validation/test splits were created for the router classifier, or how the benchmark data was preprocessed."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The Discussion & Conclusion (Section 5) is a single short paragraph that mentions future work but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper — neither specific nor generic."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit statements about what the results do NOT show. The paper mentions 'Future work will explore extending this architecture to other domains such as code generation and legal reasoning' (which contradicts having already tested code generation) but does not state scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available — no model outputs, router predictions, per-example results, or supplementary data files."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "The benchmarks used are named (GSM8K, MBPP, HumanEval), but the router training data collection is not described: how were ensemble outputs gathered, how were ground-truth labels assigned, what was the training set composition?"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw model outputs to ensemble features (semantic agreement, output variance) to router training to final evaluation is not documented. Feature extraction methodology is mentioned conceptually but not specified in reproducible detail."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": false,
         "answer": false,
         "justification": "The author is listed as 'Independent Researcher' with a personal email (arindamkhaled@gmail.com). This is clearly unfunded solo work."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The author's affiliation is disclosed as 'Independent Researcher.' No product from the author's organization is being evaluated — the paper uses open-source models from Meta, Mistral AI, and Alibaba."
       },
       "funder_independent_of_outcome": {
         "applies": false,
         "answer": false,
         "justification": "The work is unfunded (solo independent researcher)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the four models used (Llama-3-8B, Mistral-7B, Qwen-2.5-7B, Llama-3-70B), despite evaluating them on benchmarks."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (GSM8K, MBPP, HumanEval) appeared in the models' training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), MBPP (2021), and HumanEval (2021) all predate the training of Llama-3, Mistral-7B, and Qwen-2.5-7B. These models almost certainly saw these benchmarks during training, yet contamination is not discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports relative cost metrics: 61% cost reduction, 3.5x compute cost reduction at T=0.70, and latency overhead of +0.82s. While absolute costs are not reported, the relative cost analysis is central to the paper's contribution."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated — no GPU hours, hardware specifications, training time for the router classifiers, or total API/compute spend."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from single runs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results are presented without indicating how many runs produced them."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for either the XGBoost or Random Forest router classifiers, nor for the threshold selection."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1 explains the empirical motivation for choosing XGBoost over Random Forest for math reasoning, but does not describe a systematic selection process (e.g., validation set performance comparison, cross-validation)."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Multiple comparisons are made across benchmarks and configurations but no significance tests are performed at all, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement the baselines (FrugalGPT cascade) and the proposed system. No acknowledgment of author-evaluation bias or independent evaluation."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 4 shows the Pareto frontier of accuracy vs. computational cost at different routing thresholds, directly presenting performance as a function of compute budget."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether GSM8K, MBPP, or HumanEval actually measure the capabilities the paper claims to optimize. For example, GSM8K math problems may not represent real-world 'hard reasoning' that would benefit from Oracle escalation."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The routing architecture IS the thing being tested. The paper evaluates its routing framework as a bundled system, not making claims about isolated model capabilities."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "GSM8K (2021), MBPP (2021), and HumanEval (2021) all predate the models' training. No discussion of temporal leakage — the models may have seen these benchmark solutions during training."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, whether the router's training data and the benchmark evaluation data are properly separated is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of independence between the router's training data and the evaluation test set, or between benchmark examples."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are used (no canary strings, membership inference, decontamination, or temporal splits)."
       }
     }
   },
   "claims": [
     {
       "claim": "Pyramid MoA achieves 93.0% accuracy on GSM8K, effectively matching the Oracle baseline (98.0%) while reducing compute costs by 61%.",
       "evidence": "Abstract and Introduction state these figures. However, Section 3.3.1 reports 91.4% accuracy at T=0.70 with 25.5% escalation. The 93.0%/61% operating point is asserted but not shown in the detailed results section.",
       "supported": "weak"
     },
     {
       "claim": "The Consensus Router achieves 82.6% recall on MBPP, intercepting 19 out of 23 buggy code snippets.",
       "evidence": "Section 3.2.1 and Figure 2a show the confusion matrix. N=23 buggy samples is very small for generalizable claims.",
       "supported": "moderate"
     },
     {
       "claim": "Output Length Variance and Semantic Agreement are the dominant predictive features, outperforming self-reported model confidence.",
       "evidence": "Feature importance analysis in Figure 2b validates this for the Random Forest router on MBPP. Aligns with prior work on model miscalibration [9].",
       "supported": "moderate"
     },
     {
       "claim": "The Consensus MoA achieves 40% lower inference cost than FrugalGPT for the same accuracy level on HumanEval.",
       "evidence": "Figure 3 shows the comparison. However, no variance is reported and the comparison methodology is not detailed.",
       "supported": "moderate"
     },
     {
       "claim": "At T=0.70, the Anytime Router achieves 91.4% accuracy on GSM8K while escalating only 25.5% of queries, representing a 3.5x cost reduction.",
       "evidence": "Section 3.3.1 reports these figures from a held-out test set (N=1,052). Single-run result with no variance.",
       "supported": "moderate"
     },
     {
       "claim": "The system introduces negligible latency overhead (+0.82s).",
       "evidence": "Stated in the abstract only. No detailed latency measurement methodology or breakdown is presented in the paper body.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "Abstract-body discrepancy",
       "detail": "The abstract claims 93.0% accuracy and 61% cost reduction on GSM8K, but Section 3.3.1 reports 91.4% accuracy at T=0.70 with 25.5% escalation (74.5% short-circuited, not 61%). The 93.0%/61% operating point is asserted in the abstract and introduction but never documented in the detailed results."
     },
     {
       "flag": "Overclaiming: 'effectively matching'",
       "detail": "The abstract says 93.0% 'effectively matches' the Oracle's 98.0%. A 5 percentage point gap (93.0 vs 98.0) is a meaningful difference, not a match."
     },
     {
       "flag": "No error bars or variance on any result",
       "detail": "All results are single-run point estimates. For a system that depends on ensemble agreement and a trained classifier, variance across runs and random seeds is critical for interpreting whether differences are meaningful."
     },
     {
       "flag": "Tiny sample size for code generation claims",
       "detail": "The MBPP bug detection evaluation is based on only 23 buggy samples (confusion matrix in Figure 2a). The 82.6% recall (19/23) has wide confidence intervals at this sample size."
     },
     {
       "flag": "Complete absence of contamination analysis",
       "detail": "All three benchmarks (GSM8K, MBPP, HumanEval) were published in 2021, years before the models' training data was collected. The routing system's performance depends on the base models' accuracy, which may be inflated by contamination."
     },
     {
       "flag": "Self-contradictory future work",
       "detail": "Section 5 states 'Future work will explore extending this architecture to other domains such as code generation' despite Experiment I already covering code generation on MBPP/HumanEval."
     },
     {
       "flag": "Missing reproducibility details",
       "detail": "No code, no model version specifics (instruct vs base), no hyperparameters for LLM inference or router classifiers, no feature extraction methodology, no router training data description. The paper cannot be reproduced from the information provided."
     }
   ],
   "cited_papers": [
     {
       "title": "Mixture-of-Agents Enhances Large Language Model Capabilities",
       "authors": ["Junlin Wang", "Jue Wang", "Ben Athiwaratkun", "Ce Zhang", "James Zou"],
       "year": 2024,
       "arxiv_id": "2406.04692",
       "relevance": "Foundational MoA framework showing collaborative LLM layers can outperform individual models; directly extended by Pyramid MoA."
     },
     {
       "title": "Rethinking Mixture-of-Agents: Is Mixing Different Large Language Models Beneficial?",
       "authors": ["Wenzhe Li", "Yong Lin", "Mengzhou Xia", "Chi Jin"],
       "year": 2025,
       "arxiv_id": "2502.00674",
       "relevance": "Challenges MoA diversity assumptions by showing Self-MoA (single-model iteration) can match multi-model ensembles."
     },
     {
       "title": "Residual Mixture of Agents",
       "authors": ["Zhentao Xie", "Chengcheng Han", "Jinxin Shi", "Wenjun Cui", "Xin Zhao", "Xingjiao Wu", "Jiabao Zhao"],
       "year": 2025,
       "relevance": "Introduces adaptive termination and residual connections in MoA to reduce compute, achieving gains on MATH benchmark."
     },
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["Lingjiao Chen", "Matei Zaharia", "James Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Key baseline for cost-efficient LLM usage via cascading strategies; Pyramid MoA directly compares against this approach."
     },
     {
       "title": "On Calibration of Modern Neural Networks",
       "authors": ["Chuan Guo", "Geoff Pleiss", "Yu Sun", "Kilian Q. Weinberger"],
       "year": 2017,
       "relevance": "Documents model miscalibration (confident but wrong), motivating the paper's use of external agreement signals rather than self-reported confidence."
     },
     {
       "title": "BranchyNet: Fast Inference via Early Exiting from Deep Neural Networks",
       "authors": ["Surat Teerapittayanon", "Bradley McDanel", "H.T. Kung"],
       "year": 2016,
       "relevance": "Early-exit strategy for deep networks that inspired the anytime/early-termination approach in Pyramid MoA."
     },
     {
       "title": "Sparse MoA",
       "authors": ["Giang Do", "Hung Le", "Truyen Tran"],
       "year": 2024,
       "relevance": "Introduces sparsity-based early stopping in MoA architectures to reduce computational overhead."
     }
   ]
 }