ai-research-survey

scan.json (27903B)

---

 {
   "paper": {
     "title": "Learn to Code Sustainably: An Empirical Study on LLM-based Green Code Generation",
     "authors": [
       "Tina Vartziotis",
       "Ippolyti Dellatolas",
       "George Dasoulas",
       "Maximilian Schmidt",
       "Florian Schneider",
       "Tim Hoffmann",
       "Sotirios Kotsopoulos",
       "Michael Keckeisen"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2403.03344",
     "doi": "10.48550/arXiv.2403.03344"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This paper evaluates the 'green capacity' of ChatGPT-3.5, GitHub Copilot, and Amazon CodeWhisperer on 6 LeetCode problems, measuring runtime, memory, FLOPs, and energy consumption. AI tools show partial ability to improve code sustainability when explicitly asked to optimize for specific metrics, but generally produce less efficient code than top human LeetCode submissions. Copilot demonstrates the strongest optimization capability, while ChatGPT and CodeWhisperer more frequently fail to produce valid optimized code for certain problems.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'Our evaluation code is available here' (Section I.B) but no URL is visible in the text. No repository link or archive is provided."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "No dataset of generated code samples, measurement results, or test cases is released. The LeetCode problems are public but the paper's experimental data is not shared."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is specified (Intel Core i7-6700k, 32GB RAM, Linux Mint 21.2, kernel 5.17.0-79-generic, perf 5.15.111, Python 3.12.0rc1) but no requirements.txt, Dockerfile, or full dependency specification is provided."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided. The methodology section describes the process conceptually but does not provide runnable commands or scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates and heatmap values. Despite averaging 10 runtime measurements, no confidence intervals, error bars, or uncertainty ranges are reported."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "No statistical significance tests are used. Comparisons between tools and human submissions are based solely on raw metric values and Performance Delta calculations without any tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The Performance Delta (PD) metric provides relative improvements with baseline context: e.g., 'Copilot achieves a PD of 0.61 on the NDTO coding problem. This implies that Copilot's optimized code consumes 0.61 times less energy than the initially generated code' (Section IV)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "Only 6 LeetCode problems are used with no justification for why this number is sufficient. No power analysis or discussion of sample size adequacy is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states '10 runtime measurements' are averaged (Section III.D) but reports no standard deviation, IQR, or any spread measure. Only averages are presented."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three AI code generation tools are compared against each other and against human submissions (top 0.05% of LeetCode submissions by runtime)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "ChatGPT 3.5, GitHub Copilot Individual, and Amazon CodeWhisperer Individual Tier were current commercial tools at the time of writing (2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party code generation tools as black boxes rather than proposing a system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Five sustainability metrics are used: code correctness, runtime, memory usage, FLOPs, and energy consumption (Section II.B)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Code correctness is evaluated both via LeetCode's automated tests and by human analysis: 'The correctness is evaluated by LeetCode and by human evaluation' (Section III.D)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Code is submitted to LeetCode's comprehensive test suite which includes 'tests encompassing critical inputs and boundary scenarios' (Section III.D). No tuning is performed on these test cases."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per coding problem (6 problems) and per sustainability metric. Figures 2-5 show per-problem, per-tool breakdowns for GC, energy, runtime, and memory."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Failures are discussed: 'ChatGPT fails to produce a valid code output for 3 out of 6 coding problems' for runtime, and 'CodeWhisperer and ChatGPT fail to produce or optimize codes for 3 and 4 coding problems, respectively' for energy (Section IV)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Negative PD values are reported showing optimization requests sometimes make code worse. 'Copilot outputs code whose runtime is worse than the initial implementation' for some problems (Section IV). Overall finding that AI tools are less efficient than human submissions is a negative result."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'findings shed light on the current capacity of AI models to contribute to sustainable software development,' which is appropriately hedged. Results support that AI tools show partial optimization capability but lag behind human submissions."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — that requesting optimization improves sustainability metrics — is based on a controlled manipulation (initial prompt vs. optimization prompt to the same tool). This is an adequate design for this specific claim."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'LLM-based Green Code Generation' broadly, but the study tests only Python, only 6 LeetCode algorithmic problems, and only 3 specific tools. No explicit bounds are placed on generalization despite the narrow scope."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed. The paper does not consider prompt sensitivity, problem selection bias, the effect of using the tools' default interfaces, or whether the LeetCode problems are representative of real-world coding tasks."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly discusses the relationship between its metrics and actual sustainability, noting that 'FLOPs are the metric most representative of a code's sustainability' but 'the link between FLOPs and energy consumption or runtime is not always straightforward' (Section II.B). They also scope their study to code-level efficiency, excluding grid carbon intensity, PUE, and hardware variation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Only product-level versions given: 'ChatGPT version 3.5, GitHub Copilot Individual Version, and Amazon CodeWhisperer Individual Tier' (Section III.C). No API version, snapshot date, or model checkpoint identifiers are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "Only the optimization instruction pattern is provided ('Q: Give me a runtime-optimized solution for this problem'). The full task descriptions from LeetCode and function headers are referenced but not reproduced in the paper or a linked repository."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API hyperparameters (temperature, top-p, max tokens) are reported for any of the three tools. The paper does not mention any sampling or generation settings."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates ChatGPT, Copilot, and CodeWhisperer as third-party tools through their standard interfaces. The authors cannot describe internal scaffolding they have no access to."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Test case creation is documented: random input generation for problems like Search, iteration counts chosen per problem complexity (1,000 or 100,000), and the use of perf and tracemalloc for measurements are described (Section III.C-D)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion mentions future work directions (broader problems, better prompts, more metrics, hardware variability) but these are aspirations, not limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed. Issues like small problem set, contamination risk, prompt sensitivity, and platform-specific measurement artifacts are not acknowledged."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not clearly state what the results do NOT show or what populations/settings are excluded from the claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "No raw measurement data, generated code samples, or detailed per-run results are made available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The code generation process (Section III.B), measurement procedures using perf and tracemalloc (Section III.D), and hardware setup (Section III.C) are described in sufficient detail to understand data collection."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. Human baseline code comes from LeetCode's public submission pool, which is a standard platform source."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from problem selection → code generation (initial and optimized) → correctness validation via LeetCode → sustainability measurement via perf/tracemalloc → GC calculation is documented across Sections III.A-D and Figure 1."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No explicit funding source is disclosed. The acknowledgments mention 'This work inspired a development project between Mercedes-Benz and TWT' and gratitude to IBM, Microsoft, and GitHub for discussions, but no formal funding disclosure."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed: TWT GmbH, NTUA, MIT, Harvard, and Mercedes-Benz. None of the authors are affiliated with OpenAI, GitHub/Microsoft, or Amazon (the companies whose tools are evaluated)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "Funding is not disclosed, so independence cannot be assessed. The acknowledged connections to Microsoft and GitHub (whose product Copilot is evaluated) raise questions about potential conflicts."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for ChatGPT-3.5, Copilot, or CodeWhisperer. This is critical since LeetCode solutions are widely available online and almost certainly in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems and their solutions are among the most common programming benchmarks and very likely in the training data of all three models. This overlap is never discussed."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems have been publicly available for years before any of these models were trained. The paper does not address whether the models may have memorized solutions rather than generating them fresh."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The 'human submissions' are existing LeetCode solutions from the platform's public pool."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants; the study evaluates AI tools and uses publicly available LeetCode submissions."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or blinding-relevant experimental design."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or cost per code generation are reported for any of the three tools. The paper measures energy of executing generated code but not the cost of generating it."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is described (Intel i7-6700k, 32GB RAM) but no total computational budget, GPU hours, or API spend is reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. The non-deterministic nature of LLM outputs is not addressed — each tool was apparently queried once per problem per prompt variant."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The number of measurement runs is stated: 'we chose to sample 10 runtime measurements' (Section III.D) and iterations per problem are specified (1,000 or 100,000)."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is reported. Prompt wording, iteration counts, and tool configurations appear chosen without documented exploration of alternatives."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The human baseline is selected as 'the top 0.05% of LeetCode submissions based on runtime,' explicitly cherry-picking the best human code. No justification for why single prompt variants represent each tool's best capability."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The paper evaluates third-party tools (ChatGPT, Copilot, CodeWhisperer) rather than a system built by the authors, so self-comparison bias in the Lucic et al. sense does not apply."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The cost of generating code via each tool is not reported or compared. The tools may use vastly different amounts of compute for generation, but this is not discussed."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether 6 algorithmic LeetCode problems actually measure 'green code generation' capability. LeetCode problems are designed for interview practice, not sustainability assessment, and this gap is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "The three tools are evaluated as bundled products through their standard interfaces (ChatGPT web, Copilot IDE, CodeWhisperer IDE). The paper does not claim to isolate the model from the tool."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "LeetCode problems have been publicly available for years before these models were trained. The temporal overlap between training data and test problems is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether providing the full LeetCode problem description and function signature gives the models enough context to recall memorized solutions rather than generate new ones."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 6 LeetCode problems (or their solutions) appear in the models' training data. LeetCode is one of the most common sources in programming training corpora."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is used. No canary strings, membership inference, or temporal analysis applied."
       }
     }
   },
   "claims": [
     {
       "claim": "AI models demonstrate partial understanding of optimization criteria and can achieve sustainability improvements when explicitly asked to optimize for specific metrics.",
       "evidence": "Green Capacity values in Figure 2 show positive GC scores for ChatGPT and Copilot across most problems when optimization is requested (Section IV).",
       "supported": "moderate"
     },
     {
       "claim": "Human submissions (top 0.05% LeetCode) generally surpass AI-generated code in sustainability metrics.",
       "evidence": "Figure 2 shows GC_human generally exceeds GC_AI: 'selecting the runtime-optimized human submissions leads to generally surpassing the AI models in performance' (Section IV).",
       "supported": "moderate"
     },
     {
       "claim": "Runtime-optimized human code achieves improvements across multiple sustainability metrics simultaneously, suggesting correlation among metrics.",
       "evidence": "The paper states 'the same human submission can achieve improvements over multiple sustainability metrics (without explicitly optimizing them)' indicating 'a correlation among the sustainability metrics' (Section IV).",
       "supported": "weak"
     },
     {
       "claim": "Copilot demonstrates the best optimization behavior across coding problems, particularly for energy consumption.",
       "evidence": "Figure 3 shows Copilot achieves positive PD values across more problems than ChatGPT or CodeWhisperer for energy. 'Copilot showcases the best behavior across the coding problems' (Section IV).",
       "supported": "weak"
     },
     {
       "claim": "Memory was not a significant contributor to Green Capacity values, with most PD values being zero or non-positive.",
       "evidence": "Figure 5 and Section IV: 'We observe a high number of zero PD values, highlighting that memory was not a significant contributor to the Green capacity values.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Extremely small sample size",
       "detail": "Only 6 LeetCode algorithmic problems are used. This is far too few to draw general conclusions about AI tools' green coding capabilities. No justification for this sample size."
     },
     {
       "flag": "No statistical tests or uncertainty quantification",
       "detail": "Despite averaging 10 runtime measurements, no standard deviations, confidence intervals, or significance tests are reported. All comparisons are based on raw point estimates."
     },
     {
       "flag": "Contamination completely ignored",
       "detail": "LeetCode problems are among the most common programming benchmarks and are almost certainly in the training data of all three models. The paper does not acknowledge or address this, making it impossible to know whether models are generating or recalling solutions."
     },
     {
       "flag": "Cherry-picked human baseline",
       "detail": "The human baseline is the top 0.05% of LeetCode submissions — the absolute best human solutions on the platform. Comparing average AI tool output against elite human code is an unfair comparison presented without adequate framing."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations or threats-to-validity section despite significant methodological gaps including tiny sample, no contamination controls, and no statistical analysis."
     },
     {
       "flag": "Different tool interfaces confound comparison",
       "detail": "ChatGPT is used via web browser (conversational), while Copilot and CodeWhisperer are used via IDE (comment-based code generation). These fundamentally different interaction modes are not controlled for or acknowledged as a confound."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating the code quality of AI-assisted code generation tools: An empirical study on GitHub Copilot, Amazon CodeWhisperer, and ChatGPT",
       "authors": ["B. Yetistiren", "I. Özsoy", "M. Ayerdem", "E. Tüzün"],
       "year": 2023,
       "doi": "10.48550/arXiv.2304.10778",
       "relevance": "Directly evaluates code quality of the same three AI code generation tools studied in this paper."
     },
     {
       "title": "An empirical evaluation of GitHub Copilot's code suggestions",
       "authors": ["N. Nguyen", "S. Nadi"],
       "year": 2022,
       "doi": "10.1145/3524842.3528470",
       "relevance": "Empirical evaluation of GitHub Copilot code suggestions quality and correctness."
     },
     {
       "title": "A systematic evaluation of large language models of code",
       "authors": ["F. F. Xu", "U. Alon", "G. Neubig", "V. J. Hellendoorn"],
       "year": 2022,
       "doi": "10.1145/3520312.3534862",
       "relevance": "Systematic evaluation of LLMs for code generation, directly relevant to understanding code LLM capabilities."
     },
     {
       "title": "Green AI: Do deep learning frameworks have different costs?",
       "authors": ["S. Georgiou", "M. Kechagia", "T. Sharma", "F. Sarro", "Y. Zou"],
       "year": 2022,
       "doi": "10.1145/3510003.3510221",
       "relevance": "Evaluates environmental costs of deep learning frameworks, directly related to AI sustainability measurement."
     },
     {
       "title": "Energy and policy considerations for deep learning in NLP",
       "authors": ["E. Strubell", "A. Ganesh", "A. McCallum"],
       "year": 2019,
       "doi": "10.18653/v1/P19-1355",
       "relevance": "Foundational work on energy costs of training NLP models, key reference for AI sustainability research."
     },
     {
       "title": "Green AI",
       "authors": ["R. Schwartz", "J. Dodge", "N. A. Smith", "O. Etzioni"],
       "year": 2020,
       "doi": "10.1145/3381831",
       "relevance": "Defines the Green AI concept and argues for efficiency as a primary evaluation criterion alongside accuracy."
     },
     {
       "title": "Towards the systematic reporting of the energy and carbon footprints of machine learning",
       "authors": ["P. Henderson", "J. Hu", "J. Romoff", "E. Brunskill", "D. Jurafsky", "J. Pineau"],
       "year": 2020,
       "doi": "10.48550/ARXIV.2002.05651",
       "arxiv_id": "2002.05651",
       "relevance": "Proposes systematic reporting of ML energy/carbon footprints, foundational for sustainability measurement methodology."
     },
     {
       "title": "Jigsaw: Large language models meet program synthesis",
       "authors": ["N. Jain"],
       "year": 2022,
       "doi": "10.1145/3510003.3510203",
       "relevance": "LLM-based program synthesis combining language models with program synthesis techniques."
     },
     {
       "title": "Energy efficiency across programming languages: How do energy, time, and memory relate?",
       "authors": ["R. Pereira"],
       "year": 2017,
       "relevance": "Foundational study on energy efficiency of programming languages, relevant to green coding metrics."
     },
     {
       "title": "Assessing the quality of GitHub Copilot's code generation",
       "authors": ["B. Yetistiren", "E. Tüzün", "I. Özsoy"],
       "year": 2022,
       "doi": "10.1145/3558489.3559072",
       "relevance": "Quality assessment of Copilot-generated code, directly relevant to AI code generation evaluation."
     }
   ]
 }