ai-research-survey

scan.json (20671B)

---

 {
   "paper": {
     "title": "AI's Environmental Cost: Comparing Resource Consumption Between SLMs and LLMs Across Queries",
     "authors": ["Aryaanshi Sundaram", "Sparsh Kamdar", "Shreyas Kumar"],
     "year": 2025,
     "venue": "The Proceedings of the 5th International Conference on AI Research (ICAIR 2025)"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL or code archive is provided. Visualizations were done in Jupyter Notebook with Matplotlib but no code is shared."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Data was organized in a Google Drive folder and Google Sheet, but no public link or download is provided in the paper."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Jupyter Notebook and Matplotlib but provides no dependency versions, requirements file, or environment specification."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The methodology section describes the procedure narratively but provides no step-by-step reproduction instructions, scripts, or README."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results report averages and standard deviations for some consumption values but no confidence intervals or error bars are reported on the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SLMs consume 60-70% less energy and water than LLMs and makes comparative claims throughout, but no statistical significance tests are performed."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "Raw differences and 'difference factors' are reported but no formal effect sizes (Cohen's d, etc.) are provided. The percentage differences lack baseline context in a standardized form."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "48 questions with 3 repetitions each were used. No justification for why this sample size is adequate, and no power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Standard deviations are reported for power consumption across questions in several figures (e.g., Claude Puzzles LLM std dev 0.00604 kW, SLM 0.0244 kW)."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "SLMs serve as baselines against LLMs, and three commercial AI systems (ChatGPT, Claude, DeepSeek) are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models tested (GPT-4o, Claude Sonnet 4, DeepSeek R1/V3) are contemporary commercial models."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The study compares existing commercial models; there is no system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The study reports power consumption (kW), water consumption (L), and accuracy across all models and query types."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant here; accuracy is assessed against known correct answers (SAT questions with answer keys)."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a machine learning training/evaluation study; questions are used as test stimuli for commercial models, not for model development."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by subject (Math, Reading, Writing, Puzzles), by difficulty level (easy, medium, hard), and by model type (SLM vs LLM)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Specific question IDs that each model got wrong are listed in Section 4, and accuracy drops for Puzzles are discussed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that SLM accuracy drops considerably for Puzzles, and that DeepSeek's high resource consumption is due to network latency rather than computation, which undermines its environmental comparison."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims SLMs consumed 60-70% less energy/water and had same accuracy for Math/Reading but declined for Puzzles. These are supported by the results in Section 4 and Discussion."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper implies causal relationships (e.g., query complexity 'causes' higher resource consumption) but the study design cannot establish causation — response time is confounded with network latency, server load, and other factors. The DeepSeek latency issue is acknowledged but not controlled for."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame findings broadly ('AI's Environmental Cost') but the study tests only 48 text-based questions on three commercial systems measured via browser response time. The limitations section acknowledges some bounds but the title and framing overgeneralize."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section acknowledges that DeepSeek's higher resource consumption likely stems from network latency (servers in China, testing in US), and notes that internet traffic and server load could affect response times."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are named: GPT-4o, GPT-4o-mini, Claude Sonnet 4, Claude 3.5-Haiku, DeepSeek R1, DeepSeek V3."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes question categories and sources (SAT question bank, puzzles) but does not provide the actual prompts or questions used."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters (temperature, top-p, max tokens) are reported. The models were used via browser interfaces with default settings, but this is not stated explicitly."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used; questions are submitted directly to commercial chatbot interfaces."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes question selection criteria (SAT question bank with difficulty filters, random number generator for selection, exclusion of visual prompts), data organization in Google Drive/Sheets, and the formulas used to compute energy and water from response times."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "The Discussion section includes a paragraph on limitations covering network latency, exclusion of image queries, internet traffic variability, and system-specific testing differences."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: DeepSeek's server location in China causing latency inflation, exclusion of image/OCR queries limiting generalizability, internet traffic variation during testing windows, and system-specific differences (temporary chat vs. new chat)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The limitations mention exclusion of image queries but do not explicitly state what the results do NOT show. The paper does not bound its claims to specific populations, settings, or explicitly list what claims it is NOT making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Raw data is stored in a Google Drive folder and Google Sheet but no public access link is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 describes in detail the data collection procedure: browser setup, developer console timing, temporary/new chat sessions, three repetitions per question, screenshot capture, and testing window (2-5 PM)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the study tests commercial AI models with standardized questions."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from question selection → execution on models → response time recording → energy/water computation via formulas → visualization in Matplotlib is documented in Section 3."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are disclosed. The Acknowledgements thank DiscoverSTEM Innovation and Research Lab for hosting but do not mention any grants or funding."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: DiscoverSTEM (Plano, TX) and Texas A&M University. None of the authors are affiliated with the AI companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is scored as NO per schema guidance."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The study measures resource consumption and accuracy on SAT/puzzle questions; it does not evaluate model capability on a benchmark where contamination would matter. The questions test known-answer accuracy, not whether the model memorized them."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — contamination is not relevant to the environmental cost measurement, which is the primary contribution."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Same as above — the study's primary contribution is resource consumption measurement, not benchmark performance evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The paper states 'This paper did not require ethical clearance.'"
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Inference cost IS the paper's primary contribution — power consumption (kW) and water consumption (L) per query are reported for all models."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The total computational budget for running the experiments is not stated. The paper reports per-query costs but not the total resources consumed for the full experimental campaign."
       }
     }
   },
   "claims": [
     {
       "claim": "SLMs consumed 60-70% less energy and water than their LLM counterparts on average.",
       "evidence": "Section 4 and Discussion present per-query power and water consumption comparisons. E.g., ChatGPT SLM Math avg 0.0202 kW vs LLM 0.0271 kW.",
       "supported": "moderate"
     },
     {
       "claim": "SLMs had the same level of accuracy as LLMs in Math and Reading subjects.",
       "evidence": "Section 4 reports accuracy figures: all SLMs were 91.67% accurate in Reading (same as each other); Math accuracy was 100% for most models.",
       "supported": "moderate"
     },
     {
       "claim": "SLM accuracy dropped considerably for abstract reasoning tasks such as Puzzles.",
       "evidence": "Section 4: Claude SLM accuracy dropped to 50% for Puzzles vs 66% for its LLM. ChatGPT SLM was 100% accurate but this contradicts the claim.",
       "supported": "weak"
     },
     {
       "claim": "DeepSeek consumed at least nine times more power and water than other models across all subjects.",
       "evidence": "Discussion section states this explicitly; supported by data such as DeepSeek LLM Math avg 0.2839 kW vs ChatGPT 0.0202 kW and Claude 0.0188 kW.",
       "supported": "moderate"
     },
     {
       "claim": "Query complexity is positively correlated with both energy and water consumption in inference (H1).",
       "evidence": "The paper shows some spikes at harder questions but no formal correlation analysis is performed. The relationship appears inconsistent across models.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SLMs consumed 60-70% less energy and water than LLMs on average, while maintaining comparable accuracy on structured tasks like Math and Reading. However, SLM accuracy declined for abstract reasoning tasks (Puzzles). DeepSeek consumed dramatically more resources than ChatGPT and Claude, likely due to network latency from server location in China rather than computational differences. The authors propose context-aware model switching as a strategy to balance environmental cost and accuracy.",
   "red_flags": [
     {
       "flag": "Response time as proxy for energy consumption",
       "detail": "Energy and water consumption are computed entirely from browser-measured response times multiplied by assumed GPU power draw constants. Response time from a browser includes network latency, queuing, and other factors unrelated to actual compute. This is acknowledged for DeepSeek (China servers) but applies to all models — the measurements do not reflect actual energy consumption at the data center."
     },
     {
       "flag": "Extremely small sample size",
       "detail": "Only 48 questions with 3 repetitions each, across 4 subjects and 3 difficulty levels. With 4 questions per subject-difficulty cell, the statistical power to detect real differences is very low. No significance tests are performed."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Claims about differences between SLMs and LLMs, and between AI systems, are made by comparing raw numbers without any statistical testing. Given the small sample and high variance, many observed differences may not be statistically significant."
     },
     {
       "flag": "Assumed hardware parameters not validated",
       "detail": "The energy formula uses assumed Pcritical (GPU power per hour) and PUE values from literature, but these may not match the actual hardware and data center configurations used by ChatGPT, Claude, or DeepSeek. The resulting energy figures are estimates based on assumptions, not measurements."
     },
     {
       "flag": "Contradictory accuracy results",
       "detail": "The abstract claims SLM accuracy drops for complex tasks, but ChatGPT's SLM was 100% accurate on Puzzles while its LLM was only 58.33%. This directly contradicts the general narrative."
     }
   ],
   "cited_papers": [
     {
       "title": "How Hungry is AI? Benchmarking Energy, Water, and Carbon Footprint of LLM Inference",
       "authors": ["N. Jegham", "M. Abdelatti", "L. Elmoubarki", "A. Hendawi"],
       "year": 2025,
       "arxiv_id": "2505.09598",
       "relevance": "Directly relevant benchmark of LLM inference environmental costs, source of energy/water formulas used in this paper."
     },
     {
       "title": "Energy and Policy Considerations for Deep Learning in NLP",
       "authors": ["E. Strubell", "A. Ganesh", "A. McCallum"],
       "year": 2019,
       "arxiv_id": "1906.02243",
       "relevance": "Foundational work quantifying the carbon footprint of NLP model training."
     },
     {
       "title": "The growing energy footprint of artificial intelligence",
       "authors": ["A. de Vries"],
       "year": 2023,
       "doi": "10.1016/j.joule.2023.09.004",
       "relevance": "Analysis of AI's growing energy consumption and data center requirements."
     },
     {
       "title": "Towards the systematic reporting of the energy and carbon footprints of machine learning",
       "authors": ["P. Henderson", "J. Hu", "J. Romoff", "E. Brunskill", "D. Jurafsky", "J. Pineau"],
       "year": 2020,
       "relevance": "Proposes systematic reporting frameworks for ML energy and carbon footprints."
     },
     {
       "title": "Trends in AI inference energy consumption: Beyond the performance-vs-parameter laws of deep learning",
       "authors": ["R. Desislavov", "F. Martínez-Plumed", "J. Hernández-Orallo"],
       "year": 2023,
       "doi": "10.1016/j.suscom.2023.100857",
       "relevance": "Analysis of inference-time energy consumption trends in AI, relevant to environmental sustainability of LLM deployment."
     },
     {
       "title": "Are emergent abilities of large language models a mirage?",
       "authors": ["R. Schaeffer", "B. Miranda", "S. Koyejo"],
       "year": 2023,
       "arxiv_id": "2304.15004",
       "relevance": "Questions whether scaling laws reliably predict model capabilities, relevant to the SLM vs LLM efficiency debate."
     },
     {
       "title": "Factuality of large language models: A survey",
       "authors": ["Y. Wang", "M. Wang", "M.A. Manzoor", "F. Liu", "G. Georgiev", "R.J. Das", "P. Nakov"],
       "year": 2024,
       "arxiv_id": "2402.02420",
       "relevance": "Survey of LLM factuality, relevant to understanding accuracy tradeoffs between model sizes."
     },
     {
       "title": "Train large, then compress: Rethinking model size for efficient training and inference of transformers",
       "authors": ["Z. Li", "E. Wallace", "S. Shen", "K. Lin", "K. Keutzer", "D. Klein", "J.E. Gonzalez"],
       "year": 2020,
       "arxiv_id": "2002.11794",
       "relevance": "Demonstrates model compression strategies that reduce energy use while preserving accuracy, directly relevant to SLM efficiency."
     }
   ]
 }