ai-research-survey

scan.json (34312B)

---

 {
   "paper": {
     "title": "Intelligence per Watt: Measuring Intelligence Efficiency of Local AI",
     "authors": [
       "Jon Saad-Falcon",
       "Avanika Narayan",
       "Hakki Orhun Akengin",
       "J. Wes Griffin",
       "Herumb Shandilya",
       "Adrian Gamarra Lafuente",
       "Medhya Goel",
       "Rebecca Joseph",
       "Shlok Natarajan",
       "Etash Kumar Guha",
       "Shang Zhu",
       "Ben Athiwaratkun",
       "John Hennessy",
       "Azalia Mirhoseini",
       "Christopher Ré"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2511.07885",
     "doi": "10.48550/arXiv.2511.07885"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly releases a profiling harness: 'We release our IPW profiling harness to enable intelligence-per-watt benchmarking' with a GitHub URL at https://github.com/HazyResearch/intelligence-per-watt."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets are publicly available: WildChat, NaturalReasoning, MMLU Pro, and SuperGPQA. The paper references these public sources and describes the sampling procedure."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes hardware platforms in detail (Table 9) and inference hyperparameters (App B.1) but does not provide a requirements.txt, Dockerfile, or detailed software dependency list with library versions. Software tools (NVML, powermetrics, ROCm SMI) are named but not versioned."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the released profiling harness presumably contains instructions, the paper itself does not include step-by-step reproduction instructions. The experimental setup is described at a high level but lacks a 'Reproducing Results' section with specific commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": true,
         "justification": "Tables 2, 3, and 4 all report ± values (e.g., '23.2 ± 1.9%', '7.92 ± 0.32 × 10^-4'). Uncertainty is consistently reported for the main metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes many comparative claims (e.g., 'B200 achieves 1.40× higher intelligence per watt than the M4 MAX') based solely on comparing numbers with ± ranges. No formal significance tests (p-values, t-tests, etc.) are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are consistently reported with baseline context throughout: '5.3× overall increase', '3.1× improvement in accuracy', 'from 23.2% to 71.3%', percentage reductions (80.4% energy, 77.3% compute, 73.8% cost). The reader can assess magnitude."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study uses 500K queries from WildChat, 500K from NaturalReasoning, 12K from MMLU Pro, and 26.5K from SuperGPQA. These sizes are never justified — no power analysis or explanation for why these specific counts were chosen."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "App B.1 states 'we execute each query 10 times and aggregate power measurements across runs. For each query, we compute the mean power draw (watts) and mean energy consumption (joules) per query by averaging across these 10 independent executions.' The ± values in tables reflect this variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares local models against frontier cloud models (Claude Sonnet 4.5, Gemini 2.5 Pro, GPT-5) and includes historical baselines (Mixtral-8x7B, Llama-3.1-8B) for longitudinal comparison. Cloud-only deployment serves as the resource consumption baseline."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Cloud baselines are state-of-the-art as of October 2025: Claude Sonnet 4.5, Gemini 2.5 Pro, GPT-5 (2025-08-07). Local model baselines include the latest releases from Qwen3, GPT-OSS, Gemma3, and IBM Granite 4.0."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper decomposes IPW gains by holding accelerator fixed while varying model generation (3.1× from model advances) and holding model fixed while varying accelerator (1.7× from hardware advances), as shown in Table 2 and Figure 5. App C.4 ablates model precision (FP16/FP8/FP4)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper defines four complementary efficiency metrics (APW, PPW, APJ, PPJ) and also reports accuracy, latency, throughput, TTFT, energy consumption, memory usage, and more (Table 8 lists the full set)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated: LLM-as-a-judge (GPT-4O) for WildChat and NaturalReasoning, exact match for MMLU Pro and SuperGPQA. No human evaluation of model outputs is performed. Given claims about 'intelligence' and real-world query handling, human validation of output quality would strengthen the findings."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses standard benchmark test sets (MMLU Pro, SuperGPQA) and samples from WildChat and NaturalReasoning datasets. There is no model tuning on these sets — the models are evaluated in their released forms."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Extensive per-category breakdowns provided: by 22 economic domains (Tables 6, 7, Figure 7), by task type (chat vs reasoning), by difficulty level (Figures 8, 9), by model family, and by hardware accelerator."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies where local models struggle: technical domains like Architecture & Engineering (40.8-68% solvability), reasoning tasks (24pp gap vs chat), Level 5 unsolved problems (only 4.72% solved, App C.1), and specific domain weaknesses in Table 7."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results reported: local accelerators are 1.4× less efficient than cloud (Tables 3-4), SambaNova SN40L achieves 6.5-7.4× higher efficiency than Apple M4 Max (Table 4), hardest reasoning problems remain largely unsolved (Level 5, App C.1), and reasoning improvements are slower on hard problems than easy ones."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims — 88.7% query coverage (Figure 2), 5.3× IPW improvement (Table 2), 23.2% to 71.3% coverage increase (Table 2), 1.4× local vs cloud gap (Table 3) — are all directly supported by results in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper decomposes IPW gains by holding one variable fixed (accelerator or model) while varying the other, providing a controlled single-variable analysis. Table 2 isolates model contributions (3.1×) from hardware contributions (1.7×). This controlled decomposition is adequate for the causal claims made."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The study evaluates only single-turn queries, yet the title ('Intelligence per Watt: Measuring Intelligence Efficiency of Local AI') and conclusions ('local inference can meaningfully redistribute demand from centralized infrastructure') suggest general applicability to all LLM inference. Multi-turn conversations, long-context tasks, coding, tool use, and agentic workflows are not tested but not explicitly excluded from the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper presents its interpretation of findings without substantively discussing alternatives. For example, the 88.7% coverage claim depends on LLM-as-a-judge evaluation, but judge bias toward certain model families is not discussed. The IPW metric's sensitivity to the choice of accuracy measure is not explored. Inline caveats about power measurement accuracy (10-15%) are methodological notes, not alternative explanations."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures benchmark accuracy and LLM-judge win rates but frames these as 'intelligence' in the metric name (IPW). The gap between 'accuracy on selected benchmarks' and 'intelligence' is never acknowledged. The metric also reduces 'intelligence' to a scalar, ignoring task difficulty, reasoning depth, and other dimensions. The framing significantly exceeds the granularity of the measurement."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Local models are named with family and size (Qwen3-4B, GPT-OSS-120B, etc.), which is acceptable for open models with single releases. However, the LLM judge is 'GPT-4O' and the annotator is 'GPT-4O-MINI' — both marketing names without snapshot dates or API versions. Frontier baselines include 'CLAUDE SONNET 4.5' and 'GEMINI 2.5 PRO' without version identifiers. Per schema: marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in App B.1 for both LLM-as-a-judge evaluations (WildChat and NaturalReasoning) and for the economic category annotation. The actual text used is reproduced verbatim."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "App B.1 specifies: 'temperature = 0.6, top-p = 0.95, top-k = 20, min-p = 0.0, and a 32768-token output limit.' Additional settings: repetition penalty of 1.1 and length penalty of 1.0 for Qwen models. Deliberative prompting enabled for reasoning tasks."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper performs direct model inference with standard prompt-response evaluation."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "App B.1 documents preprocessing steps: filtering non-English entries from WildChat, removing malformed/nonsensical queries via GPT-4O-MINI, removing duplicates, filtering queries exceeding 32,000 characters, and filtering NaturalReasoning queries without ground truth answers."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "The paper has no dedicated Limitations or Threats to Validity section. Some limitations are mentioned inline (e.g., power measurement accuracy of 10-15% in Section 4.2), but there is no substantive dedicated discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The inline mention of 10-15% power measurement inaccuracy (Section 4.2) is specific but isolated. No systematic discussion of threats like LLM-judge reliability, benchmark representativeness, single-turn limitation generalizability, or whether the 10-run averaging adequately captures variance."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper states it focuses on 'single-turn interactions' and models '≤20B active parameters' but does not explicitly state what the results do NOT show. Multi-turn conversations, coding tasks, long-context queries, tool use, and agentic workflows are all excluded from evaluation but this is never stated as a boundary on the conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The profiling harness is released for generating new data, but the raw telemetry measurements (per-query power readings, latency traces, temperature logs from the actual study) are not explicitly released. The paper says 'We release our IPW profiling harness' — the tool, not the raw data."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Detailed description of telemetry collection: vendor APIs (NVML, powermetrics, ROCm SMI), 50ms sampling interval, per-GPU power aggregation, nanosecond timestamp synchronization. Query sources and sizes specified. App B.1 provides implementation details for each hardware platform."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public benchmarks (WildChat, NaturalReasoning, MMLU Pro, SuperGPQA). The data sources are well-documented."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: (1) query curation with filtering stages described in App B.1, (2) model inference with specified hyperparameters, (3) telemetry collection via vendor APIs with 10 repetitions per query, (4) evaluation via LLM-as-a-judge or exact match, (5) metric computation. The profiling harness comprises three described components."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive acknowledgments section listing NIH, NSF, DEVCOM ARL, ONR, Stanford HAI, and many industry partners: Google DeepMind, Google Research, Google Cloud, IBM, Microsoft, Intel, Qualcomm, Anthropic, Meta, Together, and more."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Stanford CS and Together AI. Together AI provides model serving infrastructure, and Stanford's lab receives funding from companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Multiple funders have direct stakes in the outcomes: Together AI (co-author affiliation, provides model serving), Google (makes Gemma models and cloud accelerators evaluated), IBM (Granite models evaluated), Anthropic (Claude evaluated as baseline), NVIDIA (accelerators evaluated). These companies benefit from findings showing either local inference viability (chip/device makers) or cloud superiority (cloud providers)."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement appears in the paper. Given the extensive industry funding and co-authors from Together AI, a financial interests declaration would be expected."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates pre-trained models (Qwen3, GPT-OSS, Gemma3, etc.) on benchmarks but never states the training data cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark examples (MMLU Pro, SuperGPQA, NaturalReasoning) appeared in the training data of the evaluated models. Given these are public benchmarks and the models are trained on web data, this is a significant omission."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "MMLU Pro and its predecessor MMLU have been widely available online since before the training cutoffs of all evaluated models. NaturalReasoning sources from web content. No contamination analysis is performed or discussed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Detailed cost analysis throughout. Table 12 provides per-token pricing for all models. Figures 6 shows cumulative cost over 24 hours. Energy consumption (joules per query) and power draw (watts) are reported for all model-accelerator pairs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports per-query compute metrics but never states the total computational budget consumed by this study. Evaluating 20+ models × 8 accelerators × 1M+ queries × 10 repetitions is enormous, but the total GPU hours, energy consumed, or cost of conducting the study is not reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results are averaged over 10 repetitions for power measurements, but there is no analysis of sensitivity to random seeds for model generation (temperature = 0.6 introduces stochasticity). The ± values reflect measurement variance, not seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "App B.1 explicitly states: 'we execute each query 10 times and aggregate power measurements across runs. For each query, we compute the mean power draw (watts) and mean energy consumption (joules) per query by averaging across these 10 independent executions.'"
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses specific inference hyperparameters (temperature=0.6, top-p=0.95, top-k=20) without justifying these choices or reporting whether alternative configurations were evaluated. Given that temperature and sampling parameters can significantly affect output quality and energy consumption, this is relevant."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper evaluates and reports results across all model-accelerator configurations rather than selecting a single best. The 'oracle routing' and 'best-of-local' are clearly defined selection criteria, and all individual model results are also shown (Figure 2, Tables 3-4)."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors evaluate models from companies that fund their lab (Google/Gemma, IBM/Granite, Together/GPT-OSS) using their own profiling harness and their own LLM-as-a-judge setup. The bias of evaluating systems from funders/collaborators is not acknowledged or discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "This is the central contribution of the paper. Performance is systematically reported as a function of power (IPW, Tables 2-3), energy (IPJ, Table 4), and compute. Figures 5 and 10 track efficiency trends over time. The decomposition in Table 2 explicitly controls for compute differences."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 'intelligence' in its core metric name (Intelligence Per Watt) but measures benchmark accuracy and LLM-judge win rates. The gap between 'accuracy on WildChat/NaturalReasoning/MMLU Pro/SuperGPQA' and 'intelligence' is never discussed. No analysis of whether these benchmarks measure what is claimed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models are evaluated via direct inference without scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training data for the 2025 models (Qwen3, GPT-OSS, Gemma3) includes MMLU Pro content (published 2024), NaturalReasoning content (published 2025), or SuperGPQA content (published 2025). Temporal relationships between benchmark creation and model training are not analyzed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. For example, the LLM-as-a-judge prompt includes reference answers alongside model outputs, but the impact of judge prompt design on evaluation fairness across model families is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "NaturalReasoning queries are sourced from the web, and the evaluated models are trained on web data. The potential for train-test overlap due to shared web sources is not discussed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention methods are used. No canary strings, membership inference, n-gram overlap analysis, or decontamination pipelines are applied."
       }
     }
   },
   "claims": [
     {
       "claim": "Local LMs can successfully answer 88.7% of single-turn chat and reasoning queries when routing to the best local LM per query.",
       "evidence": "Figure 2 shows best-of-local ensemble coverage across WildChat (97.8%), NaturalReasoning (88.3%), SuperGPQA (77.0%), and MMLU Pro (92.4%). Evaluation on 1M+ queries across 20+ models.",
       "supported": "strong"
     },
     {
       "claim": "Intelligence per watt improved 5.3× from 2023-2025, driven by compounding model (3.1×) and hardware (1.7×) advances.",
       "evidence": "Table 2 tracks IPW from Mixtral-8x7B on Quadro RTX 6000 (7.92×10⁻⁴) to GPT-OSS-120B on Apple M4 Max (4.18×10⁻³). Decomposition holds one variable fixed to isolate contributions.",
       "supported": "strong"
     },
     {
       "claim": "Local model coverage on single-turn queries increased from 23.2% to 71.3% from 2023 to 2025.",
       "evidence": "Table 2 shows best local LM success rate progression. Figure 3 confirms with temporal analysis of SOTA local model releases on WildChat and NaturalReasoning.",
       "supported": "strong"
     },
     {
       "claim": "Oracle routing between local and cloud models reduces energy consumption by 80.4%, compute by 77.3%, and cost by 73.8%.",
       "evidence": "Figure 6 shows simulated 24-hour workload of 80.2M queries with oracle routing vs cloud-only baseline. Based on per-query energy/compute/cost measurements from Tables 3-4 and pricing from Table 12.",
       "supported": "moderate"
     },
     {
       "claim": "A realistic router with 80% accuracy captures approximately 80% of oracle gains (64.3% energy, 61.8% compute, 59.0% cost reduction).",
       "evidence": "Figure 6 shows routing simulation results. Misrouted queries default to the cloud frontier model, maintaining task quality.",
       "supported": "moderate"
     },
     {
       "claim": "Local accelerators achieve at least 1.4× lower intelligence per watt than cloud accelerators running identical models.",
       "evidence": "Table 3 shows Apple M4 Max vs NVIDIA B200 across Qwen3 models: B200 achieves 1.40× higher IPW. Table 4 extends to intelligence per joule: B200 is 1.6-2.3× better, SambaNova SN40L is 6.5-7.4× better.",
       "supported": "strong"
     },
     {
       "claim": "FP4 quantization yields 3-3.5× energy reduction with approximately 2.5 percentage points accuracy degradation per precision step.",
       "evidence": "Figure 11 and App C.4 show systematic evaluation across 8 models on 3 datasets (N=10,000 each). Example: Qwen3-14B on SuperGPQA drops from 54.5% (FP16) to 49.0% (FP4) with 3.23× energy reduction.",
       "supported": "strong"
     },
     {
       "claim": "Chat queries are substantially more amenable to local processing than reasoning queries, with a 24 percentage point gap.",
       "evidence": "Section 5.1: best local LM achieves 88.9% coverage on WildChat vs 64.9% on NaturalReasoning. Confirmed across domains in Figure 7 and difficulty analysis in App C.1.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "Local LMs (≤20B active parameters) can handle 88.7% of single-turn chat and reasoning queries when routing to the best model per query, with intelligence per watt improving 5.3× from 2023-2025. This improvement decomposes into 3.1× from model architecture advances and 1.7× from hardware improvements. Local accelerators remain 1.4× less power-efficient than cloud accelerators for identical models, but hybrid local-cloud routing can reduce energy, compute, and cost by 60-80% while maintaining answer quality. Coverage varies substantially by domain, exceeding 90% for creative tasks but dropping to 68% for technical fields.",
   "red_flags": [
     {
       "flag": "Inflated metric framing",
       "detail": "The core metric 'Intelligence Per Watt' equates benchmark accuracy and LLM-judge win rates with 'intelligence.' This framing significantly overclaims what is actually measured (task-specific accuracy on selected benchmarks). The paper never acknowledges this gap."
     },
     {
       "flag": "Undisclosed conflicts of interest",
       "detail": "The paper is funded by many companies whose products are evaluated (Google/Gemma, IBM/Granite, Together AI/GPT-OSS, NVIDIA/accelerators, Anthropic/Claude). Two co-authors are from Together AI. No competing interests statement is provided, and the conflict is not acknowledged despite directly evaluating funders' products."
     },
     {
       "flag": "LLM-as-a-judge validity unexamined",
       "detail": "All WildChat evaluation uses GPT-4O as judge comparing outputs against Qwen3-235B reference answers. Judge bias toward certain model families (e.g., rewarding stylistic similarities to GPT/Qwen outputs) could systematically affect coverage measurements. The reference model (Qwen3-235B) is from the same family as several evaluated models. No judge validation or inter-rater agreement analysis is provided."
     },
     {
       "flag": "Complete contamination silence",
       "detail": "The paper evaluates 2025 models on benchmarks (MMLU Pro, SuperGPQA) that were publicly available during model training, yet never mentions contamination. If models have seen benchmark questions during training, the 88.7% coverage claim is inflated."
     },
     {
       "flag": "Single-turn scope with broad claims",
       "detail": "The study exclusively evaluates single-turn queries but makes sweeping claims about 'redistributing inference demand from centralized infrastructure.' Real-world LLM usage includes multi-turn conversations, coding, tool use, and agentic workflows — none tested — which may be precisely the queries least suited to local models."
     },
     {
       "flag": "Oracle routing is unrealizable",
       "detail": "The headline resource savings (80.4% energy, 77.3% compute, 73.8% cost) assume oracle routing with perfect query-to-model assignment. While the paper also shows realistic router results, the oracle numbers are prominently featured and represent a theoretical upper bound that cannot be achieved in practice."
     }
   ],
   "cited_papers": [
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Directly relevant as a cost/efficiency-aware LLM routing approach, one of the foundational works in the query routing space that IPW extends."
     },
     {
       "title": "RouteLLM: Learning to route LLMs with preference data",
       "authors": ["I. Ong", "A. Almahairi", "V. Wu", "W.-L. Chiang", "T. Wu", "J. E. Gonzalez", "M. W. Kadous", "I. Stoica"],
       "year": 2024,
       "relevance": "Key prior work on LLM routing using preference-trained classifiers, demonstrating 85% cost reduction with quality maintenance."
     },
     {
       "title": "Minions: Cost-efficient collaboration between on-device and cloud language models",
       "authors": ["A. Narayan", "D. Biderman", "S. Eyuboglu", "A. May", "S. Linderman", "J. Zou", "C. Ré"],
       "year": 2025,
       "arxiv_id": "2502.15964",
       "relevance": "Local-cloud LM collaboration protocol from the same research group, directly relevant to the hybrid inference paradigm studied."
     },
     {
       "title": "Small language models are the future of agentic AI",
       "authors": ["P. Belcak", "G. Heinrich", "S. Diao"],
       "year": 2025,
       "arxiv_id": "2506.02153",
       "relevance": "Argues for small LMs in agentic AI settings, directly complementary to this paper's empirical evidence that small models can handle most queries."
     },
     {
       "title": "RouterBench: A benchmark for multi-LLM routing system",
       "authors": ["Q. J. Hu", "J. Bieker", "X. Li"],
       "year": 2024,
       "arxiv_id": "2403.12031",
       "relevance": "Benchmark for evaluating LLM routing policies along cost-quality tradeoffs, relevant to the routing evaluation methodology."
     },
     {
       "title": "Green AI",
       "authors": ["R. Schwartz", "J. Dodge", "N. A. Smith", "O. Etzioni"],
       "year": 2020,
       "doi": "10.1145/3381831",
       "relevance": "Foundational paper proposing energy as a first-class metric for AI evaluation, directly inspiring the IPW metric."
     },
     {
       "title": "From words to watts: Benchmarking the energy costs of large language model inference",
       "authors": ["S. Samsi", "D. Zhao", "J. McDonald"],
       "year": 2023,
       "arxiv_id": "2310.03003",
       "relevance": "Prior work on LLM inference energy benchmarking that this paper extends with more models, hardware, and the accuracy-efficiency unification."
     },
     {
       "title": "Energy considerations of large language model inference and efficiency optimizations",
       "authors": ["J. Fernandez", "C. Na", "V. Tiwari"],
       "year": 2025,
       "arxiv_id": "2504.17674",
       "relevance": "Contemporary work on LLM inference energy optimization, complementary to this paper's energy profiling methodology."
     },
     {
       "title": "Which economic tasks are performed with AI? Evidence from millions of Claude conversations",
       "authors": ["K. Handa", "A. Tamkin", "M. McCain"],
       "year": 2025,
       "arxiv_id": "2503.04761",
       "relevance": "Provides the Anthropic Economic Index taxonomy used in this paper to categorize queries by occupational domain."
     },
     {
       "title": "Chatbot arena: An open platform for evaluating LLMs by human preference",
       "authors": ["W.-L. Chiang", "L. Zheng", "Y. Sheng"],
       "year": 2024,
       "relevance": "LM evaluation platform (LMArena) used to determine SOTA open-source model ranking for selecting the WildChat reference model."
     },
     {
       "title": "NaturalReasoning: Reasoning in the wild with 2.8M challenging questions",
       "authors": ["W. Yuan", "J. Yu", "S. Jiang"],
       "year": 2025,
       "arxiv_id": "2502.13124",
       "relevance": "Major reasoning benchmark used in this study, providing 500K reasoning-focused queries for evaluation."
     },
     {
       "title": "A survey on collaborative mechanisms between large and small language models",
       "authors": ["Y. Chen", "J. Zhao", "H. Han"],
       "year": 2025,
       "arxiv_id": "2505.07460",
       "relevance": "Surveys collaboration strategies between large and small LMs including pipeline, routing, distillation, and fusion approaches."
     }
   ]
 }