ai-research-survey

scan-v5.json (27727B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "Intelligence per Watt: Measuring Intelligence Efficiency of Local AI",
     "authors": [
       "Jon Saad-Falcon",
       "Avanika Narayan",
       "Hakki Orhun Akengin",
       "J. Wes Griffin",
       "Herumb Shandilya",
       "Adrian Gamarra Lafuente",
       "Medhya Goel",
       "Rebecca Joseph",
       "Shlok Natarajan",
       "Etash Kumar Guha",
       "Shang Zhu",
       "Ben Athiwaratkun",
       "John Hennessy",
       "Azalia Mirhoseini",
       "Christopher Ré"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.07885",
     "doi": "10.48550/arXiv.2511.07885"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Key abstract claims—88.7% query coverage, 5.3× IPW improvement, 1.4× local/cloud efficiency gap—are directly supported by Table 2, Figure 2, and Table 3 respectively.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Model vs. accelerator contributions are decomposed by holding one factor fixed (e.g., model at GPT-OSS-120B while varying accelerator), a standard controlled comparison appropriate for engineering benchmarking.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The 88.7% coverage figure supports broad claims about redistributing LLM inference demand, but the study is limited to single-turn interactions; multi-turn, agentic, and long-context workloads are excluded without systematic qualification in conclusions.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether WILDCHAT performance gains could reflect bias from using QWEN3-235B as reference (potentially favoring QWEN-family local models), nor whether benchmark saturation or contamination explains 2025 improvements.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "WILDCHAT accuracy is measured as win-rate against QWEN3-235B outputs yet the paper claims local models 'successfully answer' queries; this conflates beating a reference LM with actual correctness without substantive discussion of the distinction.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists; the single-turn focus and 10-15% software power measurement inaccuracy are mentioned briefly in passing, not systematically treated.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "The only specific threat acknowledged is 10-15% software power measurement error; no discussion of LLM-judge bias, reference model selection effects, benchmark contamination, or sampling representativeness.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "Single-turn focus is noted as a design choice but the paper does not explicitly state what the findings do NOT generalize to (e.g., multi-turn conversations, agentic workflows, real-time streaming).",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive acknowledgments cover NIH, NSF, DARPA (ARL), ONR, Stanford HAI, Google DeepMind, IBM, Microsoft, Anthropic, Together AI, and others.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Authors are affiliated with Stanford University and Together AI, both disclosed in the paper header.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Together AI is a co-author institution providing cloud infrastructure evaluated in the study; corporate funders IBM, Anthropic, Google, and OpenAI have products (GRANITE, Claude Sonnet 4.5, Gemini 2.5 Pro, GPT-5) directly compared in experiments.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement is included; funding sources are disclosed but individual financial interests (equity, patents, consulting) are not declared.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Intelligence per watt, local LMs (≤20B active parameters), local vs. cloud accelerators, and the routing function are all formally defined in Section 3 with mathematical notation.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Three explicit contributions are stated: the IPW metric, empirical demonstration of 88.7% local query coverage, and hybrid routing efficiency gains.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "A main related work section and extended appendix explicitly position contributions against routing benchmarks (RouterBench, RouteLLM, RouterEval), local-cloud systems (Minions, SLED, HAT), and Green AI literature.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The IPW profiling harness is released at https://github.com/HazyResearch/intelligence-per-watt, explicitly mentioned in abstract and conclusion.",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": false,
           "justification": "Source benchmarks (WILDCHAT, NATURALREASONING) are public, but the filtered/sampled subsets used, model outputs, and telemetry data from the 1M+ query study are not explicitly released.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Hardware specs are detailed in Table 9 and decoding parameters are listed, but no requirements.txt, Dockerfile, or software dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The profiling harness is released but the paper contains no step-by-step instructions for reproducing the full study across 20+ models and 8 accelerators.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": true,
           "justification": "± values are reported for IPW and accuracy metrics in Tables 2, 3, and 4, derived from 10-run averaging per query to reduce measurement noise.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "Comparative claims (1.40× efficiency gap, 5.3× improvement) are made without formal statistical significance tests; only ± intervals are provided.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "Multiplier improvements (5.3× overall, 3.1× model, 1.7× hardware, 1.40× local/cloud gap) are reported with baseline values throughout.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 500K sample from each dataset is not justified through power analysis; the choice appears driven by dataset availability rather than analytical requirements.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": true,
           "justification": "Standard error intervals (±) are reported for main efficiency metrics; each query is run 10 times and power is averaged across runs.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Cloud frontier models (QWEN3-235B, Claude Sonnet 4.5, GPT-5, Gemini 2.5 Pro) serve as baselines for coverage and cloud-only routing comparisons.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Baselines include GPT-5-2025-08-07, Gemini 2.5 Pro, and Claude Sonnet 4.5 (September 2025)—all contemporary with the October 2025 study timeframe.",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 decomposes IPW gains by holding accelerator fixed (measuring model contribution: 3.1×) and model fixed (measuring accelerator contribution: 1.7×), effectively ablating each factor.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "The study reports accuracy per watt, accuracy per joule, perplexity per watt, perplexity per joule, latency, throughput, energy, cost, and compute across model-accelerator pairs.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": true,
           "answer": false,
           "justification": "LLM-as-judge (GPT-4O) is used for WILDCHAT evaluation rather than human raters; no human evaluation of system outputs is conducted despite claims about measuring 'intelligence' quality.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": true,
           "answer": true,
           "justification": "Results are reported on standard held-out benchmark test sets (MMLU PRO, SUPERGPQA) and a held-out 500K sample of WILDCHAT queries not used in any training.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 7 and Figure 7 provide per-category performance breakdowns across 22 Anthropic Economic Index domains for both WILDCHAT and reasoning tasks.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly analyzes failure cases: Level 5 reasoning queries remain 95% unsolvable, technical domains like Architecture & Engineering show only 68% coverage, and local models show a 24pp gap on reasoning vs. chat.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Local accelerators are shown to be 1.4-7.4× less energy-efficient than cloud accelerators; reasoning queries are substantially harder for local models; these findings are reported without minimization.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Specific model variants are named (QWEN3-235B-A22B, GPT-5-2025-08-07, GRANITE-4.0-H-TINY) with references to technical reports and release dates.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": true,
           "justification": "Full LLM-as-judge prompts for WILDCHAT and NATURALREASONING evaluation, and the query categorization prompt with all options, are included verbatim in the appendix.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Decoding parameters are fully specified: temperature=0.6, top-p=0.95, top-k=20, min-p=0.0, 32768-token output limit, repetition penalty=1.1 for QWEN models.",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; the study evaluates direct LM inference on single-turn queries.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Appendix B.1 documents filtering steps: removal of non-English entries, malformed/nonsensical queries (via GPT-4O-MINI judge), duplicates, and queries exceeding 32K characters.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": false,
           "justification": "Raw telemetry, model outputs, and the processed query subsets from the 1M+ query evaluation are not released; only the profiling harness is made available.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 4.2 and Appendix B.1 describe telemetry collection via NVML, powermetrics, and ROCm SMI at 50ms sampling intervals, with 10 runs per query for noise reduction and nanosecond-synchronized timestamps.",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants; data derives from existing public query datasets.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline from dataset curation, filtering, query categorization, model inference, LLM evaluation, to telemetry aggregation is described across Section 4 and Appendix B.",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Models are described as October 2025 releases but training data cutoffs are not stated for any evaluated model, making contamination assessment impossible.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": false,
           "justification": "No discussion of whether QWEN3, GPT-OSS, or other 2025 models may have been trained on MMLU PRO or SUPERGPQA examples.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": false,
           "justification": "MMLU PRO has been publicly available since 2024, predating all evaluated 2025 models; contamination risk is neither acknowledged nor mitigated.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": true,
           "justification": "Token pricing from OpenRouter is reported in Table 12 and cost savings (59-73% reductions) from routing scenarios are explicitly quantified in Figure 6 and Section 5.3.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Per-query FLOPs and energy are reported but the total computational cost of running the full study (20+ models × 8 accelerators × 1M+ queries × 10 runs each) is not stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Local LMs can successfully handle 88.7% of single-turn chat and reasoning queries using best-of-local routing across multiple local model families.",
       "evidence": "Figure 2 shows routing across QWEN3-4B/8B/14B and GPT-OSS-20B achieves 97.8%, 88.3%, 77.0%, and 92.4% on the four benchmarks respectively; 88.7% is the average.",
       "supported": "moderate"
     },
     {
       "claim": "Intelligence per watt improved 5.3× from 2023-2025, decomposable into 3.1× from model advances and 1.7× from hardware advances.",
       "evidence": "Table 2 shows IPW rising from 7.92×10⁻⁴ (MIXTRAL on RTX 6000) to 4.18×10⁻³ (GPT-OSS-120B on M4 Max); decomposition via controlled holdout of each factor.",
       "supported": "strong"
     },
     {
       "claim": "Cloud accelerators (NVIDIA B200) achieve 1.40× higher intelligence per watt than local accelerators (Apple M4 Max) running identical models.",
       "evidence": "Table 3 shows B200 IPW values consistently 1.40× above M4 Max across all QWEN3 model sizes from 4B to 32B.",
       "supported": "strong"
     },
     {
       "claim": "Oracle routing reduces energy by 80.4%, compute by 77.3%, and cost by 73.8% versus cloud-only deployment.",
       "evidence": "Figure 6 and Section 5.3 based on simulation of 80.2M queries over 24 hours routing between local QWEN3 models on M4 Max and QWEN3-235B on H200.",
       "supported": "moderate"
     },
     {
       "claim": "A routing system with 80% accuracy captures ~80% of oracle gains, achieving 64.3% energy reduction, 61.8% compute reduction, and 59.0% cost reduction.",
       "evidence": "Figure 6 simulation with 80% accurate router on 80.2M daily queries; assumes misrouted queries fall back to frontier model maintaining answer quality.",
       "supported": "moderate"
     },
     {
       "claim": "Locally-serviceable query coverage increased from 23.2% (2023) to 71.3% (2025) for single-turn interactions.",
       "evidence": "Table 2 year-over-year tracking: MIXTRAL-8X7B 23.2% → LLAMA-3.1-8B 48.7% → GPT-OSS-120B 71.3% on combined chat and reasoning queries.",
       "supported": "strong"
     },
     {
       "claim": "Chat queries are substantially more amenable to local processing than reasoning queries, with a 24pp gap for the best individual local model (88.9% vs 64.9%).",
       "evidence": "Section 5.1 directly compares WILDCHAT vs. NATURALREASONING coverage for best individual local LM; consistent with domain breakdowns in Table 6.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "observational"
   ],
   "key_findings": "Local LMs with ≤20B active parameters can handle 88.7% of single-turn chat and reasoning queries when using best-of-local routing across model families, with coverage exceeding 90% for creative tasks but dropping to 68% for specialized technical fields like Architecture & Engineering. Intelligence per watt improved 5.3× from 2023-2025 through compounding model (3.1×) and hardware (1.7×) advances, while locally-serviceable query coverage grew from 23.2% to 71.3%. Despite local models' growing capability, cloud accelerators (NVIDIA B200, SambaNova SN40L) remain 1.4-7.4× more energy-efficient than the Apple M4 Max for identical workloads, with the gap widening substantially on per-joule metrics due to faster completion times. Hybrid local-cloud routing with 80% routing accuracy achieves 59-64% reductions in energy, compute, and cost relative to cloud-only deployment while maintaining answer quality.",
   "red_flags": [
     {
       "flag": "WILDCHAT reference model bias",
       "detail": "WILDCHAT accuracy is measured as win-rate against QWEN3-235B outputs rather than ground truth; GPT-OSS-120B achieves 89.2% by beating another LM, not by comparison to human-verified answers. This may systematically favor QWEN-family local models since the reference is from the same developer."
     },
     {
       "flag": "Single-turn only with broad infrastructure claims",
       "detail": "The 88.7% coverage figure is used to support broad claims about redistributing inference demand, but all results exclude multi-turn, agentic, long-context, and streaming use cases that constitute substantial real-world LLM traffic."
     },
     {
       "flag": "No limitations section",
       "detail": "No dedicated limitations or threats-to-validity discussion despite multiple methodological choices (LLM-as-judge, reference model selection, single-turn scope, software power measurement) that warrant scrutiny."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "MMLU PRO has been publicly available since 2024, predating all evaluated 2025 models; the possibility that QWEN3, GPT-OSS, and others were trained on benchmark examples is not acknowledged or mitigated."
     },
     {
       "flag": "Funder-outcome conflicts",
       "detail": "Together AI is a co-author institution providing cloud infrastructure evaluated in the study; IBM, Anthropic, Google, and OpenAI are listed funders whose products (GRANITE 4.0, Claude Sonnet 4.5, Gemini 2.5 Pro, GPT-5) are directly compared in experiments."
     },
     {
       "flag": "Oracle routing headlined over realistic figures",
       "detail": "The 80.4% energy reduction is a theoretical oracle bound; the more realistic 80%-accurate-router result (64.3%) is less prominent despite being more actionable, and the 80% routing accuracy assumption is itself unvalidated in this study."
     }
   ],
   "cited_papers": [
     {
       "title": "Minions: Cost-efficient collaboration between on-device and cloud language models",
       "relevance": "Directly related prior work on local-cloud LM collaboration protocols; co-authored by same Stanford group"
     },
     {
       "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
       "relevance": "Primary standardized knowledge benchmark used in evaluation"
     },
     {
       "title": "SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines",
       "relevance": "Primary graduate-level reasoning benchmark used in evaluation"
     },
     {
       "title": "WildVis: Open Source Visualizer for Million-Scale Chat Logs in the Wild",
       "relevance": "Source of WILDCHAT—1M real-world ChatGPT queries used as naturalistic workload"
     },
     {
       "title": "RouteLLM: Learning to Route LLMs with Preference Data",
       "relevance": "Key prior work on LLM routing that this study builds upon and extends to local-cloud setting"
     },
     {
       "title": "From Words to Watts: Benchmarking the Energy Costs of Large Language Model Inference",
       "relevance": "Prior work establishing methodology for LLM energy measurement that this study's telemetry collection follows"
     },
     {
       "title": "Green AI",
       "relevance": "Foundational work calling for energy as a first-class metric alongside accuracy; motivates the IPW metric"
     },
     {
       "title": "NaturalReasoning: Reasoning in the Wild with 2.8M Challenging Questions",
       "relevance": "Primary reasoning dataset (500K sample) used for evaluation"
     },
     {
       "title": "RouterBench: A Benchmark for Multi-LLM Routing System",
       "relevance": "Prior routing benchmark that this work explicitly positions against and extends"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "Directly addresses infrastructure cost and energy for LLM deployment with actionable routing recommendations and a released profiling harness."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that small local models can handle 88.7% of queries challenges common assumptions that frontier cloud models are necessary for most tasks."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Raises energy and infrastructure concerns about AI scaling but frames them constructively as a solved or solvable problem rather than a risk."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild local-vs-cloud framing, but positioned as complementary infrastructure rather than adversarial competition."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Profiling harness is released on GitHub and can be used, but full reproduction requires access to significant hardware (NVIDIA B200, Apple M4 Max, etc.)."
     },
     "brand_recognition": {
       "score": 3,
       "justification": "Stanford, Together AI, and evaluations of GPT-5, Gemini 2.5 Pro, Claude Sonnet 4.5, QWEN3, and NVIDIA hardware ensure high brand recognition across multiple major AI organizations."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "45905451",
         "title": "LLM Output Drift in Financial Workflows: Validation and Mitigation (arXiv)",
         "points": 24,
         "comments": 26,
         "url": "https://news.ycombinator.com/item?id=45905451",
         "created_at": "2025-11-12T19:53:25Z"
       },
       {
         "hn_id": "46118400",
         "title": "Intelligence per Watt: Measuring Intelligence Efficiency of Local AI",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=46118400",
         "created_at": "2025-12-02T06:55:58Z"
       },
       {
         "hn_id": "29775107",
         "title": "Program Synthesis Performance Of GitHub Copilot vs. Genetic Programming",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=29775107",
         "created_at": "2022-01-02T23:06:39Z"
       },
       {
         "hn_id": "38285475",
         "title": "Bring Your Own KG: Self-Supervised Program Synthesis for Zero-Shot KGQA",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=38285475",
         "created_at": "2023-11-16T03:11:23Z"
       }
     ],
     "top_points": 24,
     "total_points": 32,
     "total_comments": 26
   }
 }