ai-research-survey

scan.json (24656B)

---

 {
   "paper": {
     "title": "Carbon Footprint Evaluation of Code Generation through LLM as a Service",
     "authors": ["Tina Vartziotis", "Maximilian Schmidt", "George Dasoulas", "Ippolyti Dellatolas", "Stefano Attademo", "Viet Dung Le", "Anke Wiechmann", "Tim Hoffmann", "Michael Keckeisen", "Sotirios Kotsopoulos"],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2504.01036",
     "doi": "10.48550/arXiv.2504.01036"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "No repository URL, GitHub link, or code archive is provided anywhere in the paper. The energy measurement tools (E3, Microsoft Fabric) are mentioned but no analysis code or scripts are released."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset download link or raw data release is provided. The paper reports summary statistics (Tables 1-4) but no underlying data files are made available."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions using Windows Energy Estimation Engine (E3) on a Windows laptop with battery, and Intel's computational method for server estimates, but no software versions, OS version, hardware specifications (beyond generic mention of CPU/memory), or dependency specifications are provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology is described at a high level but there are no scripts, commands, or detailed procedures that would allow replication."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as single point estimates (e.g., 1.777 kgCO2e total, 9.203 kWh embodied energy in Table 4). No confidence intervals, error bars, or uncertainty measures are provided."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims '78 hours without Copilot reduced to just over 5 hours' and that operational emissions are 'eight times less than the equivalent embodied carbon emissions' without any statistical tests. These are comparative claims supported only by point estimates."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "While the paper reports absolute numbers (1.582 kgCO2e embodied vs 0.194 kgCO2e operational), there is no formal effect size reporting. The '78 hours vs 5 hours' time savings comparison lacks baseline context for how the 78-hour estimate was derived."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The study evaluates a single project (149 frontend + 50 backend test files). There is no justification for why this sample is sufficient to draw general conclusions, nor any acknowledgment that a single case study limits generalizability."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. All measurements appear to be from single runs. No repeated measurements are mentioned."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": false,
         "justification": "The paper briefly compares LLM-generated code time (5 hours) vs manual coding time (78 hours) but does not include a proper baseline comparison for carbon footprint. There is no comparison against human-written test code's carbon footprint or against alternative LLM tools."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": false,
         "justification": "No formal baselines are included, so the question of whether they are contemporary is moot. The paper only uses GitHub Copilot without comparing to any other tools or approaches."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The system is a measurement framework applied to a single tool (GitHub Copilot). There are no components to ablate — the study measures carbon footprint of a fixed pipeline."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports multiple metrics: embodied energy, operational energy, carbon intensity, embodied carbon emissions, operational carbon emissions, total carbon emissions (Table 4), plus time savings, token counts, and consumption rates (Tables 1-2)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "The paper measures energy consumption and carbon emissions of code execution, which are objective physical measurements. Human evaluation of outputs is not relevant to these claims."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a benchmark evaluation paper. The study measures carbon footprint of a specific software project, not model performance on a test set."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides breakdowns by frontend vs backend (Tables 1, 3) and by embodied vs operational carbon (Table 4), rather than only reporting aggregate numbers."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper notes that 'tests for the backend applications had a 50% accuracy rate, which required a software engineer to correct and rewrite part of the generated code' (Section 4.2). This is an honest discussion of where the approach partially failed."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that the embodied carbon of Copilot 'can not be accurately estimated due to the lack of information' (Section 3.1), and that the backend test accuracy was only 50%, which are negative findings about the approach's limitations."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims are modest: presenting 'an overview of green coding and metrics,' introducing 'LLM as a service,' and defining 'embodied and operational carbon.' These descriptive claims are supported by the paper's content in Sections 2-4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims GitHub Copilot 'reduced' development time from 78 hours to 5 hours (Section 3.1), which is a causal claim. However, the 78-hour baseline is described as an estimate without explanation of how it was derived. No controlled comparison was conducted to support this causal claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The conclusion discusses the 'environmental implications of AI-driven code generation' broadly and mentions implications for 'industries like automotive,' but the study is based on a single project at one company using one tool. The title ('Carbon Footprint Evaluation of Code Generation through LLM as a Service') is broader than what a single case study can support."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for its findings. For example, the time savings could be partly due to task simplicity rather than Copilot effectiveness. The carbon estimates rely on Intel's simplified method without discussing how different estimation approaches might yield different results."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper refers to 'GitHub Copilot' without specifying which version or underlying model was used. No model version, snapshot date, or API version is provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes that Copilot was given 'source already written' as input to generate test modules (Table 1), but no actual prompts, prompt templates, or examples of inputs to Copilot are provided."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameters for GitHub Copilot are reported (temperature, sampling settings, etc.). The paper also does not report any configuration settings for the energy measurement tools."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper uses GitHub Copilot as a third-party black-box tool. The authors cannot be expected to describe its internal scaffolding."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not describe how the input files were selected or prepared for Copilot, how E3 energy data was filtered beyond a brief mention of filtering by process name, or what preprocessing was done on the raw measurements."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section. The paper mentions individual limitations in passing (e.g., E3 requiring battery-powered devices, inability to accurately estimate Copilot's embodied carbon) but has no structured discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. The paper does not address measurement uncertainty, the representativeness of a single project, or the limitations of using Intel's simplified estimation method."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "While Section 1.3 briefly states 'This study excludes variability in grid carbon intensity, PUE, or hardware efficiency,' the paper does not explicitly state what populations, settings, or use cases the results do NOT generalize to. The conclusion draws broad implications without acknowledging scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw data is available. Only summary tables (Tables 1-4) are provided. The raw E3 energy measurements, token logs, and timing data are not released."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The data collection procedure is described: E3 monitors energy consumption in one-minute intervals (Section 4.2), Microsoft Fabric measures capacity unit seconds (Section 3.1), and the token estimation approach using Intel's method is detailed with the formula Energyemb = P * T * N (Section 3.1)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. This is a measurement study of code generation tools applied to an existing software project."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The pipeline from raw E3 recordings to final carbon footprint numbers is only partially documented. The paper mentions filtering E3 records by process name and summing TotalEnergyConsumption values, but the full pipeline from measurement to the numbers in Table 4 is not fully traceable."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. The Acknowledgments section mentions the work is 'inspired by a development project between Mercedes-Benz and TWT' and thanks IBM, Microsoft, and GitHub, but no explicit funding disclosure is made."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: TWT GmbH, Harvard, MIT, Mercedes-Benz, and NTUA. The connection between the authors' employers (TWT and Mercedes-Benz) and the case study subject is visible."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "The work involves Mercedes-Benz (whose employees are co-authors) and TWT (employer of multiple authors), and the case study evaluates a tool used in their joint project. The Acknowledgments thank Microsoft and GitHub (makers of the evaluated tool). These relationships create potential conflicts that are not addressed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Authors from TWT and Mercedes-Benz evaluate tools used in their commercial project without a declaration of interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": false,
         "answer": false,
         "justification": "The paper does not evaluate a pre-trained model's capability on a benchmark. It measures carbon footprint of code generation in a real project setting."
       },
       "train_test_overlap_discussed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark evaluation is performed. The study measures energy consumption and carbon emissions of a specific software project."
       },
       "benchmark_contamination_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No benchmark is used. The study is a case study of carbon footprint measurement, not a model capability evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. This is a measurement study of software tools."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central contribution is reporting inference cost. Table 4 reports embodied energy (9.203 kWh), operational energy (1.131 kWh), and total carbon emissions (1.777 kgCO2e). Token counts and consumption rates are also provided."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "The total computational budget is stated: server power consumption P = 356W (350W CPU + 6W memory), token latency T = 0.47s, N = 203,717 tokens processed (Section 3.1). Total embodied energy of 9.203 kWh and operational energy of 1.131 kWh are reported."
       }
     }
   },
   "claims": [
     {
       "claim": "Using GitHub Copilot reduced development time from approximately 78 hours to just over 5 hours for the software testing case study.",
       "evidence": "Section 3.1 states this time comparison but provides no methodology for how the 78-hour manual estimate was derived.",
       "supported": "weak"
     },
     {
       "claim": "The total carbon footprint of the LLMaaS cycle for this project was 1.777 kgCO2e, with embodied carbon (1.582 kgCO2e) dominating operational carbon (0.194 kgCO2e).",
       "evidence": "Table 4 provides the detailed breakdown. The embodied energy calculation uses Intel's simplified method with explicit parameters (Section 3.1).",
       "supported": "moderate"
     },
     {
       "claim": "Operational carbon emissions are approximately eight times less than embodied carbon emissions.",
       "evidence": "Table 4 shows 0.194 kgCO2e operational vs 1.582 kgCO2e embodied. The ratio is roughly 8:1 (Section 4.2).",
       "supported": "moderate"
     },
     {
       "claim": "Frontend tests were all successfully generated by Copilot while backend tests had only a 50% accuracy rate.",
       "evidence": "Section 4.2 states this directly, though no detailed error analysis or definition of 'accuracy' is provided.",
       "supported": "moderate"
     },
     {
       "claim": "The methods and tools available to assess the environmental footprint of computing activities are inadequate.",
       "evidence": "The conclusion states this (Section 5) but it is presented as an opinion rather than a finding supported by systematic evidence.",
       "supported": "weak"
     }
   ],
   "methodology_tags": ["case-study"],
   "key_findings": "This paper introduces a taxonomy distinguishing embodied and operational carbon footprints in the 'LLM as a Service' (LLMaaS) cycle for code generation. Applied to a single case study using GitHub Copilot for software testing at Mercedes-Benz/TWT, the total carbon footprint was estimated at 1.777 kgCO2e, with embodied carbon (inference) dominating at ~89% of total emissions. Frontend test generation was fully successful while backend tests achieved only 50% accuracy, requiring manual correction.",
   "red_flags": [
     {
       "flag": "Single case study presented as general evaluation",
       "detail": "The paper evaluates one project at one company using one tool but draws broad conclusions about 'Carbon Footprint Evaluation of Code Generation through LLM as a Service' generally. The title and conclusion significantly outrun what a single case study can support."
     },
     {
       "flag": "Unvalidated baseline estimate",
       "detail": "The claim that manual development would take 78 hours (vs 5 hours with Copilot) is stated without any explanation of how the 78-hour estimate was derived. This is a critical comparison point that lacks methodological support."
     },
     {
       "flag": "Simplified estimation method for core results",
       "detail": "The embodied energy calculation uses 'Intel's simplified computational method' referenced only to 'a blog post by Intel on Medium' (footnote 5). The actual blog post URL is not provided, and the method's accuracy is not validated against actual measurements."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "All carbon footprint numbers are reported as precise point estimates (e.g., 1.777 kgCO2e) despite relying on estimated parameters (server power, token latency) and a simplified calculation method. No error bounds or sensitivity analysis is provided."
     },
     {
       "flag": "Potential conflict of interest",
       "detail": "Authors from TWT and Mercedes-Benz evaluate GitHub Copilot in a joint project, while acknowledging Microsoft and GitHub (Copilot's developer) 'for the discussions on automated code generation.' This relationship is not declared as a potential conflict."
     },
     {
       "flag": "No reproducibility artifacts",
       "detail": "No code, data, or detailed measurement logs are released. The E3 energy measurements, raw token counts, and calculation scripts are not available for independent verification."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational work on code generation with LLMs (Codex/HumanEval), directly relevant to evaluating LLM coding capabilities."
     },
     {
       "title": "Energy and policy considerations for deep learning in NLP",
       "authors": ["E. Strubell", "A. Ganesh", "A. McCallum"],
       "year": 2019,
       "relevance": "Seminal work on the energy and carbon costs of training large NLP models, foundational to the green AI movement."
     },
     {
       "title": "Carbon emissions and large neural network training",
       "authors": ["D. A. Patterson", "J. Gonzalez", "Q. V. Le"],
       "year": 2021,
       "arxiv_id": "2104.10350",
       "relevance": "Quantifies carbon emissions of training large neural networks, providing baseline data for the environmental impact discussion."
     },
     {
       "title": "LLMCarbon: Modeling the end-to-end carbon footprint of large language models",
       "authors": ["A. Faiz", "S. Kaneda", "R. Wang"],
       "year": 2024,
       "relevance": "Provides a model for estimating total carbon footprint of LLMs including embodied and operational costs, directly used in this paper's framework."
     },
     {
       "title": "Green AI",
       "authors": ["R. Schwartz", "J. Dodge", "N. A. Smith", "O. Etzioni"],
       "year": 2020,
       "doi": "10.1145/3381831",
       "relevance": "Advocates for considering computational efficiency alongside accuracy in AI research, foundational to the sustainability evaluation approach."
     },
     {
       "title": "Learn to code sustainably: An empirical study on LLM-based green code generation",
       "authors": ["T. Vartziotis", "I. Dellatolas", "G. Dasoulas"],
       "year": 2024,
       "arxiv_id": "2403.03344",
       "relevance": "Predecessor work by the same authors defining sustainability metrics for LLM-generated code, directly relevant to evaluating code quality and environmental impact."
     },
     {
       "title": "Measuring the carbon intensity of AI in cloud instances",
       "authors": ["J. Dodge", "T. Prewitt", "R. Tachet des Combes"],
       "year": 2022,
       "doi": "10.1145/3531146.3533234",
       "relevance": "Addresses measurement of AI carbon intensity in cloud computing, relevant to understanding environmental costs of AI-as-a-service."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2023,
       "relevance": "Evaluates correctness of LLM-generated code at NeurIPS, relevant to understanding quality of AI-generated code."
     },
     {
       "title": "Sustainable AI: environmental implications, challenges and opportunities",
       "authors": ["C. Wu", "R. Raghavendra", "U. Gupta"],
       "year": 2021,
       "arxiv_id": "2111.00364",
       "relevance": "Comprehensive analysis of environmental implications of AI systems including embodied carbon from hardware manufacturing."
     },
     {
       "title": "FrugalGPT: How to use large language models while reducing cost and improving performance",
       "authors": ["L. Chen", "M. Zaharia", "J. Zou"],
       "year": 2023,
       "arxiv_id": "2305.05176",
       "relevance": "Addresses cost reduction strategies for LLM usage, relevant to practical and sustainable deployment of AI coding assistants."
     },
     {
       "title": "Refactoring programs using large language models with few-shot examples",
       "authors": ["A. Shirafuji", "Y. Oda", "J. Suzuki", "M. Morishita", "Y. Watanobe"],
       "year": 2023,
       "arxiv_id": "2311.11690",
       "relevance": "Examines LLM-based code refactoring, relevant to understanding LLM contributions to code quality and sustainability."
     }
   ]
 }