ai-research-survey

scan.json (25640B)

---

 {
   "paper": {
     "title": "Characterizing LLM Inference Energy-Performance Tradeoffs across Workloads and GPU Scaling",
     "authors": [
       "Paul Joe Maliakel",
       "Shashikant Ilager",
       "Ivona Brandic"
     ],
     "year": 2025,
     "venue": "arXiv preprint",
     "arxiv_id": "2501.08219"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is mentioned anywhere in the paper. No supplementary material links are provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses four publicly available NLP benchmarks: BoolQ, HellaSwag, TruthfulQA, and NarrativeQA. All are standard public datasets referenced with citations. Models are also publicly available (Llama, Qwen)."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B specifies the hardware (NVIDIA RTX PRO 6000 Blackwell GPU, 96 GB memory), measurement infrastructure (NVML telemetry via nvidia-smi at 10ms sampling), inference precision (FP16), and torch.cuda.synchronize() usage. Section IV-C specifies model names and parameter counts. However, no requirements.txt or software dependency list is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While the experimental setup is described in detail (Section IV), there are no step-by-step reproduction instructions, scripts, or a README with commands to run. The methodology is described but not packaged for replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., '42% energy savings', '1-6% latency increase'). No confidence intervals, error bars, or uncertainty measures are reported despite each configuration being repeated three times."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., semantic features predict difficulty better than input length, DVFS achieves energy savings) but does not use any statistical significance tests. Comparisons are based solely on point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports effect sizes with baseline context throughout, e.g., '42% energy savings' relative to 2842 MHz baseline, '17.5 percentage point improvement' over length-only baseline (Table VI), energy per query from 2.9 J (1B) to 21.0 J (32B). Correlation coefficients are also reported (e.g., r = 0.002, r = -0.29)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 1,000 queries per dataset (3,817 total) and 3 runs per configuration, but no justification is given for why these sample sizes were chosen. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section IV-B states 'Each configuration is repeated three times, and we report mean values' but no standard deviation, variance, or spread measures across the three runs are reported. Only means are given."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses the maximum GPU frequency (2842 MHz) as the baseline for all DVFS comparisons (Section IV-B). For workload characterization, a length-only heuristic (51.1% accuracy) serves as baseline against semantic features (68.6% accuracy, Table VI)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are appropriate: maximum GPU frequency is the natural reference point for DVFS energy studies, and the length-based heuristic is the standard assumption the paper challenges. The models used (Llama-3.1/3.2, Qwen2.5) are contemporary."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table VI presents a feature ablation for difficulty classification: length only (51.1%), + entity density (66.6%), + causal question score (68.4%), semantic features only (68.6%). Section V-D also discusses feature selection rationale for excluding token entropy, reasoning complexity, and complexity score."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports energy consumption (joules), latency (end-to-end and per-phase), Energy-Delay Product (EDP), accuracy (for classification tasks), and ROUGE-L (for generation tasks). Multiple dimensions of the energy-performance tradeoff are captured."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a hardware benchmarking study measuring energy consumption and latency. Human evaluation is not relevant to the claims about GPU frequency scaling behavior and energy efficiency."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The difficulty classification ablation (Table VI) uses 5-fold stratified cross-validation, which provides proper train-test separation. The NLP benchmark datasets are used as-is without modification."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by model (5 models), dataset (4 benchmarks), batch size (1, 4, 8), GPU frequency (7 levels), and inference phase (prefill vs decode). Tables XI, XII, XIII, XVI provide detailed per-configuration breakdowns."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses 'Always Hard' queries (32.6%) where even 32B models struggle (Table IX), notes that model routing introduces quality tradeoffs (6.8 percentage point drop, Section VII-C), and acknowledges TruthfulQA's low scores across all models (Table VII)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that input length is a weak predictor (r = 0.002), that reasoning complexity has near-zero correlation with quality (r = -0.03) despite theoretical relevance, and that 32.6% of queries remain hard regardless of model size. Token entropy is excluded due to redundancy with length."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are well-supported: '44.5% of queries achieving comparable quality across model sizes' is in Table IX; 'decode phase dominates inference time (77-91%)' is in Table XI; '42% energy savings with only a 1-6% latency increase' is in Tables XI/XIV. All claims match the data."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about why DVFS works (memory-bound decode phase is insensitive to frequency). These are supported by controlled experiments varying a single factor (GPU frequency) while holding others constant, and by phase-level instrumentation showing the mechanism. The ablation study (Table VI) supports causal claims about feature contributions."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper is explicit about scope: 'Our goal is not to propose a new serving system or scheduling algorithm' (Section I). The threats to validity section (Section VII) states limitations to 'offline replay-based benchmarking on a single GPU' and notes it 'does not capture production serving dynamics.' The title specifies 'across Workloads and GPU Scaling' rather than making universal claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity paragraph (Section VII) discusses alternative factors: production serving dynamics (continuous batching, speculative decoding, multi-GPU), system-level overheads beyond GPU, classification accuracy for routing, and switching overhead. The paper also discusses why token entropy is correlated with length rather than being an independent predictor."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Table I and Section IV-C specify exact model names: Llama-3.2-1B, Llama-3.2-3B, Llama-3.1-8B, Qwen2.5-14B, Qwen2.5-32B. These are specific versioned model identifiers with parameter counts, not vague marketing names."
       },
       "prompts_provided": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not use prompting in the traditional sense. It runs standard NLP benchmark evaluations (log-likelihood for classification, greedy decoding for generation) on existing datasets. No custom prompts are designed."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-C specifies greedy decoding (do_sample=False, temperature=0), max 100 generated tokens, early stopping on EOS. Section IV-B specifies seven SM frequency levels (180-2842 MHz), batch sizes (1, 4, 8), FP16 precision, and NVML sampling at 10ms. The logistic regression uses L2 regularization with C=1.0."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. This is a benchmarking study that runs models directly for inference on standard datasets."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The paper documents data handling: 1,000 queries per dataset (817 for TruthfulQA), quality scores min-max normalized within each dataset before averaging (Section V-E), 5-fold stratified cross-validation with standardization (Section V-D). NER features extracted via spaCy en_core_web_sm. Difficulty labels based on dataset-median quality threshold."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section VII includes a 'Threats to validity' paragraph that discusses specific limitations of the study including offline-only evaluation, single GPU, and excluded production dynamics."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats to validity are specific to this study: 'offline replay-based benchmarking on a single GPU', 'does not capture production serving dynamics such as continuous batching, speculative decoding, or multi-GPU execution', 'Energy measurements reflect GPU consumption only and exclude system-level overheads', and limitations to English workloads."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states scope boundaries: 'Our goal is not to propose a new serving system or scheduling algorithm' (Section I), 'these findings are intended as empirical observations rather than deployment prescriptions' (Section VII), and lists what was not tested: distributed inference, heterogeneous hardware, non-English workloads."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw measurement data (power traces, latency logs, per-query results) is made available. Only aggregated statistics are reported in tables."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV-B describes the data collection procedure in detail: NVML telemetry via nvidia-smi at 10ms sampling, torch.cuda.synchronize() for timing, three repetitions per configuration, offline replay-based setup on an otherwise idle system."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard NLP benchmarks (BoolQ, HellaSwag, TruthfulQA, NarrativeQA) which are publicly available."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The data pipeline is documented: queries are loaded from benchmark datasets, characterized using semantic features (Section V), inference is run under specified configurations, power and latency are measured via NVML (Section IV-B), and results are aggregated. The study workflow is shown in Figure 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section lists funding from the Austrian Science Fund (FWF) through the Themis project (Grant DOI: 10.55776/PAT1668223), the Standalone Project Triton (Grant No. P 36870-N), the Austrian Research Promotion Agency (FFG) through Virtual Shepherd (Grant No. FO999910627), and the Vienna Science and Technology Fund (WWTF, Grant No. ESR24-053)."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: Paul Joe Maliakel and Ivona Brandic are at TU Wien (Computational Sustainability Group), Shashikant Ilager is at University of Amsterdam (Multiscale Networked Systems Group). No commercial affiliations with evaluated products."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funding comes from Austrian government science agencies (FWF, FFG, WWTF). These are public research funders with no financial stake in GPU DVFS or LLM inference energy optimization outcomes."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. The absence of such a declaration is noted."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate model capability on benchmarks in the traditional sense (testing whether the model 'knows' the answers). It measures energy consumption and latency during inference. The quality scores are secondary metrics used for workload characterization, not for evaluating model knowledge."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale as training_cutoff_stated. The study measures hardware energy-performance behavior, not model capability. Contamination would not affect the energy or latency measurements that are the primary contribution."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same rationale. The benchmarks are used as representative workloads to exercise inference, not to evaluate whether models have memorized answers. The primary outcomes (energy, latency, phase behavior) are independent of contamination."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. This is a hardware benchmarking study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "This is the core contribution of the paper. Energy consumption per query is reported in joules (e.g., Table XVI: 2.92 J for 1B to 20.97 J for 32B at baseline), energy per token (Figure 3), and latency measurements are provided across all configurations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "While per-query energy is reported (the paper's contribution), the total computational budget for running all experiments is not stated. No mention of total GPU hours, wall-clock time for the full study, or total energy consumed for the benchmarking campaign."
       }
     }
   },
   "claims": [
     {
       "claim": "Semantic query features predict inference difficulty better than input length, with input length showing near-zero correlation with quality (r = 0.002).",
       "evidence": "Table V and Figure 2 show input length has r = 0.002 correlation with quality. Table VI shows semantic features achieve 68.6% classification accuracy vs. 51.1% for length-only baseline. Entity density shows r = -0.29 correlation (Section V).",
       "supported": "strong"
     },
     {
       "claim": "44.5% of queries achieve comparable quality across all model sizes (1B-32B), suggesting they can be routed to smaller models.",
       "evidence": "Table IX shows 44.5% 'Always Easy' queries. Table X validates this classification: easy queries score 0.661 average quality vs. 0.405 for hard queries across all five models (Section V-E).",
       "supported": "strong"
     },
     {
       "claim": "The decode phase dominates inference time (77-91%) and is largely insensitive to GPU frequency.",
       "evidence": "Table XI shows decode time fraction ranges from 72.5% to 90.6%, and decode latency changes by less than ±1% across all frequency settings. Detailed phase-level measurements across 5 models and 3 batch sizes support this (Section VI-C).",
       "supported": "strong"
     },
     {
       "claim": "Reducing GPU frequency from 2842 MHz to 180 MHz achieves an average 42% energy savings with only 1-6% latency increase.",
       "evidence": "Table XI shows energy savings of 39.9-44.2% across all model-batch configurations, with end-to-end latency increases of -0.1% to +5.6%. Table XIV summarizes: 40-44% energy savings (avg 42%), +1-3% latency change (Section VI-B).",
       "supported": "strong"
     },
     {
       "claim": "Combining workload-aware model selection with phase-aware DVFS could reduce energy by up to 87%.",
       "evidence": "Table XVII estimates combined savings: weighted average 87% across all query categories. However, this is an upper-bound projection based on combining independently measured savings, not a deployed system. The paper acknowledges this is illustrative (Section VII).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "LLM inference energy consumption is dominated by the decode phase (77-91% of inference time), which is memory-bound and largely insensitive to GPU frequency scaling. Reducing GPU SM frequency from 2842 MHz to 180 MHz achieves ~42% energy savings with only 1-6% latency increase across five decoder-only models (1B-32B). Semantic query features (entity density, causal question presence) predict inference difficulty better than input length (r = -0.29 vs r = 0.002), with 44.5% of queries achieving comparable quality across all model sizes, suggesting significant potential for workload-aware model routing combined with phase-aware DVFS.",
   "red_flags": [
     {
       "flag": "No variance or uncertainty reported",
       "detail": "Despite running each configuration three times, the paper reports only mean values with no standard deviation, confidence intervals, or error bars. Given that hardware measurements can vary, the stability of the reported energy savings and latency figures cannot be assessed."
     },
     {
       "flag": "No code or raw data released",
       "detail": "The benchmarking code, scripts, and raw measurement data are not released, making independent verification or replication impossible despite the paper's emphasis on measurement-driven analysis."
     },
     {
       "flag": "Combined savings estimate is an upper bound, not validated",
       "detail": "The 87% combined energy savings claim (Table XVII) is an arithmetic combination of independently measured factors, not a validated end-to-end system. The paper acknowledges this but the headline number may overstate practical achievability."
     },
     {
       "flag": "Single GPU evaluation",
       "detail": "All experiments are on a single NVIDIA RTX PRO 6000 (Blackwell). DVFS behavior may differ substantially on other GPU architectures (A100, H100), data center GPUs, or multi-GPU setups. The paper acknowledges this limitation."
     }
   ],
   "cited_papers": [
     {
       "title": "Efficient memory management for large language model serving with PagedAttention",
       "authors": ["W. Kwon", "Z. Li"],
       "year": 2023,
       "relevance": "Foundational LLM serving system (vLLM) that the paper builds upon for understanding inference optimization."
     },
     {
       "title": "FlashAttention: Fast and memory-efficient exact attention with IO-awareness",
       "authors": ["T. Dao"],
       "year": 2022,
       "relevance": "Key inference optimization technique that reduces memory traffic, relevant to understanding compute vs memory-bound phases."
     },
     {
       "title": "Splitwise: Efficient generative LLM inference using phase splitting",
       "authors": ["P. Patel", "E. Choukse"],
       "year": 2024,
       "relevance": "Directly related work on phase-aware LLM inference optimization, splitting prefill and decode phases."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Addresses cost-aware LLM inference through model routing cascades, closely related to the workload-aware routing concept in this paper."
     },
     {
       "title": "GreenLLM: SLO-aware dynamic frequency scaling for energy-efficient LLM serving",
       "authors": ["Q. Liu", "D. Huang", "M. Zapater", "D. Atienza"],
       "year": 2025,
       "arxiv_id": "2508.16449",
       "relevance": "Directly related work on DVFS for LLM inference energy efficiency with SLO awareness."
     },
     {
       "title": "SLO-aware GPU DVFS for energy-efficient LLM inference serving",
       "authors": ["A. K. Kakolyris", "D. Masouros", "S. Xydis", "D. Soudris"],
       "year": 2024,
       "relevance": "Prior work on GPU DVFS for LLM inference with service-level objective awareness."
     },
     {
       "title": "Zeus: Understanding and optimizing GPU energy consumption of DNN training",
       "authors": ["J. You", "J.-W. Chung", "M. Chowdhury"],
       "year": 2023,
       "relevance": "Framework for understanding and optimizing GPU energy in deep learning, applicable to inference optimization."
     },
     {
       "title": "Energy and policy considerations for deep learning in NLP",
       "authors": ["E. Strubell", "A. Ganesh", "A. McCallum"],
       "year": 2019,
       "relevance": "Seminal paper on energy costs of NLP/deep learning that motivated the 'Green AI' movement."
     },
     {
       "title": "Green AI",
       "authors": ["R. Schwartz", "J. Dodge", "N. A. Smith", "O. Etzioni"],
       "year": 2020,
       "doi": "10.1145/3381831",
       "relevance": "Foundational position paper on energy-efficient AI research that motivates sustainability-focused benchmarking."
     },
     {
       "title": "MLPerf Inference benchmark",
       "authors": ["V. J. Reddi"],
       "year": 2020,
       "doi": "10.1109/ISCA45697.2020.00045",
       "relevance": "Standard inference benchmarking suite that provides context for LLM inference evaluation methodology."
     },
     {
       "title": "Accelerating LLM inference with staged speculative decoding",
       "authors": ["B. Spector", "C. Re"],
       "year": 2023,
       "arxiv_id": "2308.04623",
       "relevance": "Alternative approach to LLM inference efficiency through speculative decoding, complementary to DVFS optimization."
     }
   ]
 }