ai-research-survey

scan.json (20921B)

---

 {
   "paper": {
     "title": "A Cost-Benefit Analysis of On-Premise Large Language Model Deployment: Breaking Even with Commercial LLM Services",
     "authors": ["Guanzhong Pan", "Vishal Chodnekar", "Abinas Roy", "Haibo Wang"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.18101"
   },
   "scan_version": 2,
   "active_modules": [],
   "methodology_tags": ["theoretical"],
   "key_findings": "On-premise LLM deployment break-even periods vary dramatically by model size: sub-30B models on consumer GPUs (RTX 5090, ~$2k) break even in 0.3–3 months, medium models (70B–120B) in 2–34 months, and large models (235B+) in 4–69 months. The analysis covers 54 deployment scenarios (9 open-source models × 6 commercial APIs) and finds that pricing variability across providers creates large swings in cost-effectiveness. Small and mid-sized enterprises can achieve rapid payback with small open-source models, while large-scale deployments are economically viable only under sustained high-volume usage or when privacy/sovereignty concerns override cost.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides a playground at https://v0-ai-cost-calculator.vercel.app/ where users can apply the cost-benefit framework. This is a released tool, though source code for the analysis itself is not separately linked."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "The pricing, benchmark, and hardware data used in the analysis are presented in tables but no downloadable dataset or raw data file is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specifications, dependencies, or setup instructions are provided. The paper is primarily analytical with tables and equations."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The mathematical formulas (Equations 1-7) are given but there are no scripts or detailed instructions to reproduce the break-even calculations."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "No confidence intervals or error bars are reported. All break-even values are point estimates with no uncertainty quantification despite depending on variable inputs (electricity rates, hardware prices, throughput estimates)."
       },
       "significance_tests": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a deterministic cost-modeling exercise, not a statistical comparison. No comparative claims requiring significance tests are made."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table IV reports percentage performance differences alongside break-even months (e.g., 'EXAONE 4.0 32B vs GPT-5: 2.26 months (-2.65%)'), providing context for the cost-performance tradeoff."
       },
       "sample_size_justified": {
         "applies": false,
         "answer": false,
         "justification": "This is an analytical/theoretical paper with no experimental sample. The 54 deployment scenarios cover a systematic combination of models and APIs."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance or sensitivity analysis is reported for the break-even estimates. Inputs like electricity cost ($0.15/kWh), throughput, and hardware prices are fixed at single values with no sensitivity ranges."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Commercial API pricing from GPT-5, Claude-4 Opus, Claude-4 Sonnet, Grok-4, and Gemini 2.5 Pro serve as baselines for the cost comparison (Table I, Table IV)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The commercial models used as baselines (GPT-5, Claude-4, Grok-4, Gemini 2.5 Pro) and open-source models (Qwen3, EXAONE 4.0, Kimi-K2) are all current as of 2025."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper presents a cost model, not a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Models are compared on multiple benchmarks (GPQA, MATH-500, LiveCodeBench, MMLU-Pro) in Table I, and the cost analysis uses break-even time and token capacity as metrics."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to a cost-modeling paper."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "No ML model is trained or tested; this is a cost analysis paper."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model size category (small, medium, large) and by enterprise type (SME, medium, large) in Section VI.E and VI.F."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where on-premise deployment is NOT economically viable: large models against aggressively priced providers like Gemini 2.5 Pro can extend break-even to 5-9 years (Section VI.F.3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that large-scale deployments face 'steepest economic barriers' with break-even extending beyond 2 years, and that against some providers the economic case is unfavorable (Section VI.E, VII)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about break-even periods (few months for small, 2 years for medium, 5 years for large) are supported by Table IV results, though the abstract's phrasing slightly oversimplifies the ranges."
       },
       "causal_claims_justified": {
         "applies": false,
         "answer": false,
         "justification": "The paper makes no causal claims. It presents a deterministic cost model computing break-even points from known inputs."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The analysis uses fixed assumptions ($0.15/kWh electricity, specific GPU prices, 8h/day operation, 2:1 output/input token ratio) but presents results as generally applicable to 'organizations' without bounding to these specific assumptions. The title frames it broadly as 'On-Premise Large Language Model Deployment' rather than specifying the narrow conditions modeled."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative factors that could affect break-even: GPU depreciation, staffing costs for maintaining on-premise infrastructure, model update frequency requiring new hardware, or the possibility that API prices will continue to drop."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures break-even time in months based on token throughput matching, framing this as 'economic viability.' However, it does not acknowledge that cost parity in token generation is a proxy — real deployment economics include staffing, maintenance, downtime, model updates, and opportunity costs that are not modeled."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with parameter counts are provided: Qwen3-235B, Llama-3.3-70B, EXAONE 4.0 32B, etc. Commercial models are named (GPT-5, Claude-4 Opus/Sonnet, Grok-4, Gemini 2.5 Pro). Benchmark data is sourced from Artificial Analysis [64]."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting. It is a cost analysis using published benchmark scores and pricing data."
       },
       "hyperparameters_reported": {
         "applies": false,
         "answer": false,
         "justification": "No model inference or training is performed. The paper uses published benchmark data and pricing information."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not document how benchmark scores were selected or aggregated from Artificial Analysis, how hardware specifications and throughput estimates were derived, or how the 2:1 output/input token ratio was determined."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. The conclusion mentions future work (empirically validate estimates, include staffing/maintenance) but does not substantively discuss limitations of the current analysis."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed. The paper does not address threats such as rapidly changing pricing, GPU availability constraints, assumed throughput values possibly being optimistic, or the fixed electricity rate assumption."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what it does NOT cover. Staffing costs, maintenance, model fine-tuning costs, downtime risks, and multi-tenancy overhead are omitted from the model without explicit acknowledgment in a limitations discussion."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is provided. Benchmark scores are cited from Artificial Analysis [64] and hardware specs from manufacturer documentation, but no downloadable dataset is available."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section III describes the performance evaluation framework and model selection criteria. Benchmark data sources (GPQA, MATH-500, LiveCodeBench, MMLU-Pro) and the aggregation source (Artificial Analysis) are identified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data comes from published benchmarks and pricing information."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw benchmark scores and pricing data to break-even calculations is partially documented through equations, but intermediate steps (e.g., how throughput estimates were derived for each model-hardware combination) are not fully traced."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding information is disclosed. Two authors are affiliated with Carnegie Mellon University and two are unaffiliated, but no funding statement is provided."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: two at Carnegie Mellon University and two listed as 'Unaffiliated.' No evaluated product affiliations are present."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is provided."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate any pre-trained model's capability on benchmarks. It uses published benchmark scores from third-party sources for cost modeling."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — no model evaluation is performed by the authors."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the paper uses third-party benchmark results for cost comparison, not for evaluating model capability."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": false,
         "answer": false,
         "justification": "This is a theoretical cost-modeling paper. It reports costs of the systems it analyzes, not costs of its own method."
       },
       "compute_budget_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper is a theoretical analysis with no significant computation performed by the authors."
       }
     }
   },
   "claims": [
     {
       "claim": "Small open-source models (sub-30B) can break even with commercial APIs in 0.3–3 months on consumer-grade GPUs.",
       "evidence": "Table IV shows EXAONE 4.0 32B breaks even in 0.3–2.26 months, Qwen3-30B in 0.3–2.5 months, and Magistral Small in 0.4–3.0 months depending on the commercial API compared against. Hardware cost is ~$2k for a single RTX 5090 (Table III).",
       "supported": "moderate"
     },
     {
       "claim": "Medium-scale models (70B–120B) offer balanced economics with break-even periods of 2.3–34 months.",
       "evidence": "Table IV shows Llama-3.3-70B at 2.3–17.8 months, gpt-oss-120B at 3.9–30.9 months, GLM-4.5-Air at 4.3–34.0 months. Hardware costs range from $15k–$30k (Table III).",
       "supported": "moderate"
     },
     {
       "claim": "Large open-source models require break-even horizons of 4.3–69.3 months, making them viable only for sustained high-volume workloads.",
       "evidence": "Table IV shows Kimi-K2 at 8.7–69.3 months, GLM-4.5 at 6.5–51.5 months, Qwen3-235B at 4.3–34.0 months. Hardware costs range from $60k–$240k (Table III).",
       "supported": "moderate"
     },
     {
       "claim": "Open-weight models can deliver competitive performance within 20% of top commercial models.",
       "evidence": "Table I shows EXAONE 4.0 32B achieving 73.9% GPQA (vs GPT-5's 85.4%), 97.7% MATH-500 (vs 99.4%), and 74.7% LiveCodeBench (vs 66.8% for GPT-5). Performance gaps noted in Table IV's amortized performance difference column.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Oversimplified cost model",
       "detail": "The TCO model includes only hardware and electricity. It explicitly omits staffing costs, maintenance, cooling infrastructure, networking, software licensing, downtime, and model update/replacement costs — factors acknowledged in the future work section as needing inclusion. These omissions systematically bias toward on-premise deployment appearing more favorable."
     },
     {
       "flag": "Fixed assumptions without sensitivity analysis",
       "detail": "Key inputs are fixed at single values: electricity at $0.15/kWh, 8 hours/day operation, 20 days/month, 2:1 output/input token ratio. No sensitivity analysis explores how results change under different assumptions (e.g., 24/7 operation, different electricity rates, different usage patterns)."
     },
     {
       "flag": "Throughput estimates not empirically validated",
       "detail": "Token throughput values in Table III (e.g., 200 tok/sec for EXAONE 4.0 32B on RTX 5090, 400 tok/sec for Qwen3-235B on 4×A100) are presented without citation or empirical measurement. These values critically affect break-even calculations but their source and accuracy are unclear."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any dedicated discussion of limitations or threats to validity, despite the analysis depending heavily on assumptions that may not hold in practice."
     },
     {
       "flag": "Benchmark scores from third-party aggregator",
       "detail": "All benchmark performance data comes from Artificial Analysis [64], a third-party aggregator, rather than from the original papers or the authors' own measurements. The reliability and methodology of this source is not discussed."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Directly relevant to LLM deployment economics — proposes strategies for reducing API costs through model cascades and caching."
     },
     {
       "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention",
       "authors": ["W. Kwon"],
       "year": 2023,
       "arxiv_id": "2309.06180",
       "relevance": "vLLM's PagedAttention is a key inference optimization that makes on-premise LLM deployment more feasible."
     },
     {
       "title": "Inference Economics of Language Models",
       "authors": ["E. Erdil"],
       "year": 2025,
       "arxiv_id": "2506.04645",
       "relevance": "Directly models inference economics for LLMs including hardware amortization and energy efficiency."
     },
     {
       "title": "An Inquiry into Datacenter TCO for LLM Inference with FP8",
       "authors": ["J. Kim"],
       "year": 2025,
       "arxiv_id": "2502.01070",
       "relevance": "Analyzes datacenter total cost of ownership for LLM inference, considering precision formats."
     },
     {
       "title": "Energy Considerations of Large Language Model Inference and Efficiency Optimizations",
       "authors": ["J. Fernandez", "C. Na", "V. Tiwari", "Y. Bisk", "S. Luccioni", "E. Strubell"],
       "year": 2025,
       "arxiv_id": "2504.17674",
       "relevance": "Analyzes energy costs and efficiency optimizations for LLM inference, relevant to deployment economics."
     },
     {
       "title": "Large Language Models: A Survey",
       "authors": ["S. Minaee", "T. Mikolov"],
       "year": 2024,
       "arxiv_id": "2402.06196",
       "relevance": "Comprehensive LLM survey covering capabilities and deployment considerations."
     },
     {
       "title": "Position: On-Premises LLM Deployment Demands a Middle Path: Preserving Privacy Without Sacrificing Model Confidentiality",
       "authors": ["H. Huang"],
       "year": 2024,
       "arxiv_id": "2410.11182",
       "relevance": "Directly addresses the privacy vs. capability tradeoff in on-premise LLM deployment."
     },
     {
       "title": "Qwen3 Technical Report",
       "authors": ["A. Yang"],
       "year": 2025,
       "arxiv_id": "2505.09388",
       "relevance": "Technical report for one of the key open-source models evaluated in this cost analysis."
     }
   ]
 }