ai-research-survey

scan.json (23541B)

---

 {
   "paper": {
     "title": "Beyond Benchmarks: The Economics of AI Inference",
     "authors": ["Boqin Zhuang", "Jiacheng Qiao", "Mingqian Liu", "Mingxing Yu", "Ping Hong", "Rui Li", "Xiaoxia Song", "Xiangjun Xu", "Xu Chen", "Yaoyao Ma", "Yujie Gao"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2510.26136"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper proposes an 'economics of inference' framework evaluating 9 LLMs on WiNEval-3.0 (2,993 medical tasks) across cost, performance, and quality. It finds diminishing marginal cost with increasing concurrency, an optimal concurrency inflection point per model, and identifies WiNGPT-3.5 (the authors' own model) as the best cost-quality option at $0.34/76.2 score. The framework estimates A800 GPU hourly cost at ~$0.79 and maps a cost-quality Pareto frontier.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, code archive, or link to source code is provided anywhere in the paper."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 is described as a proprietary medical evaluation set from Winning Health. No download link or public access is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'A800 80G × 2 cards' but provides no software environment details — no inference framework version, OS, driver version, or dependency specifications."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No reproduction instructions, README, or scripts are provided. The benchmark is proprietary and unavailable."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are point estimates. No confidence intervals or error bars are reported in any table or figure."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims WiNGPT-3.5 is the 'overall leader' and makes comparative claims across models, but no statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports absolute cost and score values with enough context to compute differences (e.g., WiNGPT-3.5 at $0.34/76.2 vs Seed-OSS-36B at $0.55/72.2). Appendix B shows performance changes across concurrency levels with absolute values."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 2,993 requests but no justification is given for why this number is sufficient. No power analysis or discussion of statistical adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges 'slight fluctuations' from model randomness and vLLM scheduling but reports no standard deviation, variance, or multiple-run statistics. Single-run results only."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Nine models are compared including WiNGPT variants, Qwen3-30B, GLM-4-32B, Mistral-Small, medgemma-27b, Seed-OSS-36B, and gpt-oss-20b (Table 2)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models include Qwen3-30B, GLM-4-32B-0414, medgemma-27b, and Mistral-Small, which are contemporary as of 2025."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted. The framework has multiple components (cost model, performance metrics, quality score) but none are ablated."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three performance metrics (Total Completion Time, Avg TTFT, Avg Throughput) plus quality score and cost are reported (Table 1, Table 2)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper benchmarks inference economics — human evaluation of model outputs is not the focus. Quality is measured via WiNEval-3.0 automated scoring."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 is described only as a test set. No discussion of held-out vs. development splits, or whether any models were tuned on this data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 covers '10 core scenarios' (medical licensing exams, clinical diagnosis, etc.) but no per-category breakdown of scores is provided. Only aggregate scores appear in Table 2."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No failure cases or error analysis are discussed. No examples of model failures on specific tasks."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that increasing concurrency beyond the optimal point causes throughput to drop sharply and TTFT to increase (Section 6.1 point 2). WiNGPT-3.0's extreme cost ($3.47) is discussed honestly as an outlier."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about diminishing marginal cost, diminishing returns to scale, and an optimal cost-effectiveness zone are supported by the concurrency data in Appendix B and the Pareto frontier in Figure 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes causal claims like 'increasing concurrency is the most effective way to amortize fixed overhead' (Section 6.1) without controlling for confounds. The concurrency-cost relationship is presented as causal without experimental isolation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims to provide 'the first quantifiable decision-making tool for selecting the best AI technology within a limited budget' (Section 8) but all results are from a single medical benchmark (WiNEval-3.0) on A800 GPUs. The title 'The Economics of AI Inference' is far broader than what was tested."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered. For example, the paper does not discuss whether WiNGPT-3.5's high score could be due to training on similar medical data, or whether tokenizer efficiency differences confound cost comparisons."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 score is used as a proxy for 'quality' and 'intelligence' without acknowledging the gap. The paper equates benchmark performance with clinical utility: 'the core quality metric to measure a model's comprehensive abilities in medical knowledge understanding, clinical reasoning, and instruction following' (Section 5.2) but does not discuss whether benchmark scores translate to real clinical value."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed as 'WiNGPT-3.5', 'Qwen3-30B', 'GLM-4-32B-0414', etc. GLM includes a version suffix but most models lack snapshot dates or exact version identifiers. Parameter counts are given but not specific checkpoints."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "No prompts or system instructions used in the WiNEval-3.0 evaluation are provided. The evaluation setup is described only at a high level."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference hyperparameters (temperature, top-p, max tokens) are reported. Only concurrency levels are varied."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct inference."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "No description of how WiNEval-3.0 tasks were constructed, selected, or preprocessed. The benchmark's composition is described only at a high level ('10 core scenarios')."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Limitations' lists five specific limitations including exclusion of training costs, hardware dependency, proxy nature of benchmarks, lack of statistical confidence analysis, and capital expenditure considerations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: 'Changing the GPU, inference engine, or quantization strategy could significantly alter the performance and cost data' (Limitation 2), and 'WiNEval-3.0 serves as a high-quality proxy metric... not entirely equivalent to a model's final performance in specific, specialized clinical business scenarios' (Limitation 3)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 explicitly states training costs are not included (Limitation 1), results depend on specific hardware (Limitation 2), and upfront capital expenditure is not considered (Limitation 5)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. WiNEval-3.0 is proprietary and not released. Only aggregated results in tables are provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 is described as covering '10 core scenarios' from 'real clinical applications' but the actual data collection procedure is not documented."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data is a benchmark of medical tasks, not a human study."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from task execution to cost calculation is outlined (Sections 3-4) but the construction of WiNEval-3.0 itself and any filtering or curation steps are not documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding disclosure or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are identified as 'WiNGPT Team' from 'Winning Health AI Research' in the header. The affiliation is clear."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Winning Health is the developer of the WiNGPT model family. They are directly evaluating their own commercial products using their own proprietary benchmark, and their model (WiNGPT-3.5) is declared the winner. The funder/employer has a direct financial interest in the outcome."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is present. The authors work for the company whose products are being favorably evaluated."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the models tested."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WiNEval-3.0 data could have appeared in any model's training data. Particularly concerning for WiNGPT models which are developed by the same company."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 is a proprietary benchmark from Winning Health, and WiNGPT models are also from Winning Health. The obvious contamination risk — that WiNGPT may have been trained or validated on WiNEval data — is never addressed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost is the central topic. Table 2 reports cost per model for the full test set. Section 3 derives GPU hourly cost (~$0.79/hour for A800)."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified as A800 80G × 2 cards. Total execution times per model and concurrency level are reported in Appendix B."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No multiple-seed results reported. The paper acknowledges 'model generation has some inherent randomness' (Appendix B) but does not run multiple seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. Results appear to be from single runs."
       },
       "hyperparameter_search_budget": {
         "applies": false,
         "answer": false,
         "justification": "No hyperparameter search is conducted. Only concurrency levels are varied, which is a system configuration rather than model hyperparameter tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.1 explicitly states the selection criterion: 'find the concurrency setting with the lowest cost (i.e., shortest total completion time) while meeting the performance baselines (e.g., throughput > 20 tokens/s and latency < 1s).' All configurations are shown in Appendix B."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Winning Health evaluates their own WiNGPT models on their own WiNEval-3.0 benchmark and finds WiNGPT-3.5 is the 'overall leader.' No acknowledgment of self-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Performance as a function of concurrency (a compute variable) is reported for every model in Appendix B. The Pareto frontier (Figure 1) plots quality vs. cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "WiNEval-3.0 is introduced as a 'professional evaluation set for the medical field' with representative properties, but no construct validity analysis is provided. The paper does not discuss whether WiNEval-3.0 scores actually measure clinical utility."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are evaluated via direct inference."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of when WiNEval-3.0 tasks were created relative to model training periods."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For WiNGPT models tested on their own company's benchmark, this is particularly relevant."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether WiNEval-3.0 tasks overlap with WiNGPT training data or share structural similarities."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "WiNGPT-3.5 is the overall leader in cost-quality tradeoff at $0.34 cost and 76.2 quality score",
       "evidence": "Table 2 shows WiNGPT-3.5 achieves the highest score (76.2) at a competitive cost ($0.34) among all tested models.",
       "supported": "weak"
     },
     {
       "claim": "Increasing concurrency reduces total time until a performance inflection point, after which throughput drops sharply",
       "evidence": "Section 6.1 and Appendix B show concurrency-performance curves for all models. WiNGPT-3.5 drops from 2034s at concurrency 8 to 774s at concurrency 48, then throughput degrades beyond that.",
       "supported": "moderate"
     },
     {
       "claim": "A800 80G baseline hourly cost is approximately $0.79/hour under common assumptions",
       "evidence": "Appendix A provides the cost breakdown: depreciation ($0.64) + power ($0.08) + maintenance ($0.06) = $0.78/hour, with parameter assumptions documented.",
       "supported": "moderate"
     },
     {
       "claim": "This is the first 'LLM Inference Production Frontier'",
       "evidence": "The abstract claims this is the 'first' such frontier. No evidence is provided that prior work has not done similar cost-quality analysis.",
       "supported": "unsupported"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating its own product as the winner",
       "detail": "Winning Health evaluates their own WiNGPT model family on their own proprietary WiNEval-3.0 benchmark and concludes WiNGPT-3.5 is the 'overall leader.' This is an undisclosed conflict of interest with no mitigation — the benchmark, the models, and the evaluation are all controlled by the same entity."
     },
     {
       "flag": "Proprietary, unavailable benchmark",
       "detail": "WiNEval-3.0 is not publicly available, making independent verification impossible. Since the benchmark creator also makes the winning model, results cannot be validated externally."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All results are single-run point estimates with no error bars, confidence intervals, or multi-run variance, despite the paper acknowledging randomness in model generation and inference scheduling."
     },
     {
       "flag": "Potential training data contamination",
       "detail": "WiNGPT models may have been trained or validated on WiNEval-3.0 data since both come from Winning Health. This critical concern is never addressed."
     },
     {
       "flag": "Overclaiming in title and conclusion",
       "detail": "The paper claims to present 'the first quantifiable decision-making tool' for AI technology selection and 'The Economics of AI Inference' broadly, but all evidence comes from a single medical benchmark on one GPU type."
     }
   ],
   "cited_papers": [
     {
       "title": "Language models are few-shot learners",
       "authors": ["Tom Brown", "Benjamin Mann", "Nick Ryder"],
       "year": 2020,
       "relevance": "Foundational GPT-3 paper establishing LLM capabilities and scaling properties."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "arxiv_id": "2307.09288",
       "relevance": "Open-weight LLM family relevant to inference cost and deployment studies."
     },
     {
       "title": "Judging LLM-as-a-judge with MT-Bench and Chatbot Arena",
       "authors": ["Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng"],
       "year": 2023,
       "relevance": "LLM evaluation methodology and benchmarking approaches."
     },
     {
       "title": "Carbon emissions and large neural network training",
       "authors": ["David Patterson", "Joseph Gonzalez", "Quoc Le"],
       "year": 2021,
       "arxiv_id": "2104.10350",
       "relevance": "Cost and environmental impact of large model training, related to inference economics."
     },
     {
       "title": "Scaling laws for neural language models",
       "authors": ["Jared Kaplan", "Sam McCandlish", "Tom Henighan"],
       "year": 2020,
       "arxiv_id": "2001.08361",
       "relevance": "Foundational scaling laws relating compute, data, and model performance."
     },
     {
       "title": "Training compute-optimal large language models",
       "authors": ["Jordan Hoffmann", "Sebastian Borgeaud", "Arthur Mensch"],
       "year": 2022,
       "arxiv_id": "2203.15556",
       "relevance": "Chinchilla scaling laws for compute-optimal training, directly relevant to inference cost tradeoffs."
     },
     {
       "title": "Efficient memory management for large language model serving with PagedAttention",
       "authors": ["Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang"],
       "year": 2023,
       "relevance": "vLLM inference framework used in many deployment setups; relevant to inference cost optimization."
     }
   ]
 }