ai-research-survey

scan.json (27291B)

---

 {
   "paper": {
     "title": "On Evaluating the Efficiency of Source Code Generated by LLMs",
     "authors": [
       "Changan Niu",
       "Ting Zhang",
       "Chuanyi Li",
       "Bin Luo",
       "Vincent Ng"
     ],
     "year": 2024,
     "venue": "FORGE '24 (AI Foundation Models and Software Engineering)",
     "arxiv_id": "2404.06041",
     "doi": "10.1145/3650105.3652295"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The ability to generate correct code is not positively correlated with the ability to generate efficient code: GPT-4 has higher Pass@10 than GPT-3.5 but produces slower code on HumanEval and MBPP. Larger model size within a family (Code Llama, WizardCoder) does not significantly improve runtime efficiency. Chain-of-thought prompting (generating code first, then optimizing) yields larger speedups on more complex LeetCode problems (up to 1.18x) compared to simple efficiency-requesting prompts, though all prompting effects are modest on entry-level benchmarks.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We also make code, data and other artifacts available online [1]' with reference [1] pointing to https://github.com/NougatCA/EfficiencyEval."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper claims to release code, data, and artifacts online. HumanEval and MBPP are publicly available standard benchmarks, and the newly constructed LeetCodeEval is included in the artifact release."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No environment specification (requirements.txt, Dockerfile, library versions) is provided in the paper. They mention using the gem5 CPU simulator but do not specify software environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. A GitHub link is given but the paper itself lacks instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 2, 3, and 4 report only point estimates (normalized runtime, Pass@10, speedup) with no confidence intervals or error bars despite repeated measurements."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims ('code generated by the former is not as efficient as the latter') based solely on comparing numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main RQ1 results (Table 2, Table 3) report normalized runtimes and Pass@10 without contextualizing the magnitude of differences. The speedup ratios in Table 4 (RQ2) provide relative magnitudes but the primary comparative claims lack formal effect sizes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for the number of problems (164 HumanEval, 399 MBPP, 166 LeetCodeEval), the number of models tested, or k=10 generations. No power analysis is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper states 'we repeat the execution of each piece of code for 10 times and take the average runtime' and 'repeat the submission of each piece of code for 3 times and record the average results' but never reports standard deviation or any spread measure for these repeated runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple LLMs are compared against each other: GPT-4, GPT-3.5, Phi-2, Code Llama (7B/13B/34B), WizardCoder (7B/13B/34B), and DeepSeek Coder (base/instruct). Each serves as a baseline for the others."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The models evaluated (GPT-4, GPT-3.5, Code Llama, WizardCoder, DeepSeek Coder, Phi-2) were all state-of-the-art or near-SOTA at the time of writing (late 2023/early 2024)."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper comparing existing models, not proposing a system with components to ablate. The RQ2 prompting experiments compare different strategies but are not ablations of a single system."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports Pass@10 (correctness), average normalized runtime (efficiency), and percentage beats (LeetCode-specific efficiency). Multiple complementary metrics are used."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "Human evaluation is not relevant to the paper's claims about code execution efficiency, which is measured automatically via runtime."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses established benchmarks (HumanEval, MBPP) and a newly constructed LeetCodeEval as evaluation sets. No tuning or selection is performed against these benchmarks — models are evaluated as-is."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by benchmark (HumanEval vs MBPP vs LeetCodeEval) and by difficulty level within LeetCodeEval (easy, medium, hard). Table 2 and Table 3 show separate results per dataset."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Failed problems are simply excluded from runtime analysis. The paper notes overlap counts (70 problems all pass on HumanEval, 0 on hard LeetCodeEval) but does not discuss what kinds of problems models fail on or why."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports that prompting has minimal effect on simple benchmarks (HumanEval/MBPP speedups near 1.0), that DeepSeek Coder shows almost no improvement from prompting (1.00-1.01x on some benchmarks), and that no model could solve enough hard LeetCode problems for comparison."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims that efficiency is evaluated, that simple prompts help basic problems, and that chain-of-thought helps complex problems. These are supported by Tables 2-4 in the paper."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'training strategy and data have an impact on the efficiency of the generated code' based on comparing DeepSeek Coder base vs instruct, but this is a single observational comparison with many confounding differences between the two versions. No controlled experiment isolates training strategy."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims to evaluate 'Source Code Generated by LLMs' broadly, but HumanEval/MBPP are Python-only and LeetCodeEval is C++-only. Claims about 'LLMs' are based on a limited set of models. These scope limitations are not explicitly bounded in the claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss alternative explanations for key findings. For example, why GPT-3.5 generates more efficient code than GPT-4 could relate to RLHF training, code verbosity preferences, or other factors — none of which are discussed."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper equates 'efficiency' with runtime but does not acknowledge that efficiency could encompass memory usage, energy consumption, or code maintainability. The proxy gap between 'runtime' and the broader 'efficiency' framing is not discussed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The paper specifies exact API model IDs: 'gpt-3.5-turbo-1106 and gpt-4-1106-preview'. Open-source models are identified with specific sizes (Phi-2 2.7B, Code Llama 7B/13B/34B, WizardCoder 7B/13B/34B, DeepSeek Coder 33B base/instruct)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 provides the LeetCodeEval prompt template, and Figure 3 provides the three prompting strategies for RQ2. For HumanEval/MBPP, they reference Liu et al.'s published source code for prompt generation. The fill values (problem descriptions, constraints) come from publicly available benchmarks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, max tokens, or other generation hyperparameters are reported for any model. Only k (number of generations: 10 for HumanEval/MBPP, 3 for LeetCodeEval) is stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The paper evaluates direct LLM code generation from prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "For LeetCodeEval, the paper documents the filtering pipeline: select problems from May 2023+, filter out problems with images, filter out problems with more downvotes than upvotes, divide by difficulty. Table 1 provides dataset statistics."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 'THREATS TO VALIDITY' discusses potential data leakage and unstable runtime as threats, with specific mitigation strategies described."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The threats section discusses study-specific issues: (1) data leakage because training data contents are unknown, mitigated by temporal filtering for LeetCodeEval, and (2) unstable runtime, mitigated by gem5 simulator and repeated runs."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show. There is no discussion of scope limitations such as generalizability to other programming languages, problem types, or non-tested models."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'We also make code, data and other artifacts available online [1]' at a GitHub repository (https://github.com/NougatCA/EfficiencyEval), suggesting raw data is available for verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "For LeetCodeEval, the paper describes collection of URLs, titles, descriptions, examples, constraints, and code templates from LeetCode, with specific filtering criteria (post-May 2023, no images, positive vote ratio). HumanEval and MBPP are standard benchmarks with known provenance."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (HumanEval, MBPP) and a publicly available programming platform (LeetCode)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in Section 2.1.2: generate k responses → test correctness → select first passing code → measure runtime via gem5 (10 repetitions) for HumanEval/MBPP, and generate 3 codes → submit to LeetCode → record correctness and runtime (3 repetitions) for LeetCodeEval."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgments section lists 'Cooperation Fund of Huawei-NJU Creative Laboratory for the Next Programming, CCF-Huawei Populus Grove Fund, NSF award 2034508.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: Nanjing University, Singapore Management University, and University of Texas at Dallas. None of the authors are affiliated with the companies whose models are evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Huawei funds the research but none of Huawei's own LLM products are evaluated. The paper evaluates OpenAI, Meta, Microsoft, and DeepSeek models. NSF is a government funder with no stake in results."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions GPT-4's knowledge cutoff as May 2023 for LeetCodeEval construction but acknowledges 'we are unable to get the data cut-offs for the other model.' Training cutoffs are not systematically stated for all evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 states: 'Potential data leakage is a threat to construct validity because we can not know if the data used for evaluation is present in the training data of models.' The paper acknowledges this risk explicitly."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper takes a concrete mitigation step: constructing LeetCodeEval with problems from May 2023+ (after GPT-4's training cutoff) specifically to reduce contamination risk. However, contamination for HumanEval and MBPP (published 2021) is only acknowledged, not mitigated."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or inference latency are reported despite using commercial APIs (GPT-3.5, GPT-4) and generating thousands of code samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper uses the gem5 CPU simulator and OpenAI APIs but does not quantify the total compute resources used."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No random seed sensitivity analysis is reported. The paper generates k responses per problem but does not discuss seed variation or its effect on results."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'we repeat the execution of each piece of code for 10 times' on gem5 and 'repeat the submission of each piece of code for 3 times' on LeetCode."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Generation parameters (temperature, top-p) are not even stated, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of k=10 for HumanEval/MBPP and k=3 for LeetCodeEval is not justified. The selection of 'first passing code' for efficiency evaluation is described but the rationale for this specific selection strategy is not discussed."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed at all, so the question of correction for multiple comparisons does not arise."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss potential evaluation bias. While they evaluate third-party models rather than their own system, they do not acknowledge potential biases in their evaluation methodology (e.g., prompt template design favoring certain models)."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models of vastly different compute requirements (2.7B Phi-2 vs GPT-4) without discussing or normalizing for computational cost. No performance-vs-compute analysis is provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper argues that HumanEval/MBPP have limited test cases that don't adequately reveal efficiency differences, motivating LeetCodeEval: 'Comprehensive test cases on LeetCode can make the runtime benefits of code with real less complexity more significant, and thus more accurately reflect the efficiency.' This is an explicit construct validity argument."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding or agentic tools are involved. Models are evaluated via direct prompting."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly addresses temporal leakage by constructing LeetCodeEval with problems from 'May 2023 and later (this is the latest GPT-4 knowledge cut-off)' to avoid training data contamination."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information through prompt context, code templates, or test case design."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether HumanEval, MBPP, or LeetCode problems share structural similarities with training data beyond temporal overlap."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Temporal splitting is used as a concrete leakage prevention method for LeetCodeEval: only problems posted after the model's training cutoff are included."
       }
     }
   },
   "claims": [
     {
       "claim": "The ability to generate correct code is not positively correlated with the ability to generate efficient code.",
       "evidence": "Table 2: GPT-4 has higher Pass@10 (98.2 HumanEval, 94.2 MBPP) but higher normalized runtime (8.61, 9.14) than GPT-3.5 (87.2, 88.7 Pass@10; 8.35, 8.86 runtime). Phi-2 has the lowest Pass@10 but competitive runtime.",
       "supported": "moderate"
     },
     {
       "claim": "Larger number of parameters does not promise higher code efficiency performance.",
       "evidence": "Table 2: Code Llama 7B/13B/34B have similar runtimes (9.95/9.87/9.93 HumanEval; 9.58/9.61/9.54 MBPP). WizardCoder shows slight improvement with size but the pattern is inconsistent.",
       "supported": "moderate"
     },
     {
       "claim": "Training strategy and data have an impact on the efficiency of generated code.",
       "evidence": "Table 2: DeepSeek Coder 33B Instruct (7.54 runtime) significantly outperforms Base version (9.40 runtime) on HumanEval. Section 2.1.5 attributes this to instruct-tuning.",
       "supported": "weak"
     },
     {
       "claim": "Chain-of-thought prompting yields larger speedups on complex problems than simple efficiency-requesting prompts.",
       "evidence": "Table 4: On LeetCodeEval Medium, Prompt 3 achieves 1.18x speedup for GPT-4 vs 1.07x for Prompt 1. On HumanEval, all prompts yield similar modest speedups (~1.04-1.06x).",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 generates the most efficient code on more complex problems (LeetCodeEval).",
       "evidence": "Table 3: GPT-4 has the lowest runtime (30.89 easy, 50.92 medium) and highest percentage beats (65.51% easy, 73.09% medium) on LeetCodeEval.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or statistical tests",
       "detail": "All comparisons across models and prompting strategies are based on point estimates only. The paper repeats runs (10x for gem5, 3x for LeetCode) but never reports variance, and makes comparative claims without significance tests."
     },
     {
       "flag": "Severely limited comparison set on harder problems",
       "detail": "On LeetCodeEval, only 24/44 easy problems have all models passing, only 3/85 medium problems have all three models passing, and 0/33 hard problems. The medium and hard comparisons are based on tiny or empty sample sizes."
     },
     {
       "flag": "First-passing-code selection introduces bias",
       "detail": "Among k=10 (or k=3) generated solutions, the paper uses 'the first passing code' for efficiency evaluation. This selection strategy could systematically favor certain solution patterns (e.g., simpler implementations that appear first) and conflates generation order with efficiency."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, and other generation parameters are not reported for any model. These significantly affect output diversity and quality, making replication impossible."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval, the primary benchmark used in this paper and a key benchmark for evaluating LLM code generation."
     },
     {
       "title": "Program Synthesis with Large Language Models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP, the second primary benchmark used in this evaluation of LLM code generation."
     },
     {
       "title": "GPT-4 Technical Report",
       "authors": ["OpenAI"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4, one of the primary models evaluated in this code efficiency study."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Introduces Code Llama, one of the open-source code LLM families evaluated for code efficiency."
     },
     {
       "title": "WizardCoder: Empowering Code Large Language Models with Evol-Instruct",
       "authors": ["Ziyang Luo", "Can Xu", "Pu Zhao"],
       "year": 2023,
       "arxiv_id": "2306.08568",
       "relevance": "Introduces WizardCoder, evaluated in this paper for code generation efficiency."
     },
     {
       "title": "Learning Performance-Improving Code Edits",
       "authors": ["Aman Madaan", "Alexander Shypula", "Uri Alon"],
       "year": 2024,
       "relevance": "Proposes the PIE dataset for code optimization and evaluation approaches adapted by this paper for prompting strategies (Prompts 2 and 3)."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Provides the evaluation framework and source code used for generating LLM responses on HumanEval and MBPP."
     },
     {
       "title": "Self-refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "arxiv_id": "2303.17651",
       "relevance": "Self-refinement approach for LLMs applied to code optimization, related to the iterative prompting strategies evaluated."
     },
     {
       "title": "Evaluating the code quality of ai-assisted code generation tools: An empirical study on github copilot, amazon codewhisperer, and chatgpt",
       "authors": ["Burak Yetiştiren", "Işık Özsoy", "Miray Ayerdem", "Eray Tüzün"],
       "year": 2023,
       "relevance": "Evaluates code quality of LLM-based tools (Copilot, CodeWhisperer, ChatGPT), complementary to this paper's efficiency focus."
     },
     {
       "title": "Llama 2: Open foundation and fine-tuned chat models",
       "authors": ["Hugo Touvron", "Louis Martin", "Kevin Stone"],
       "year": 2023,
       "relevance": "Foundation model for Code Llama, one of the model families evaluated for code generation efficiency."
     }
   ]
 }