ai-research-survey

scan.json (34232B)

---

 {
   "paper": {
     "title": "How Efficient is LLM-Generated Code? A Rigorous & High-Standard Benchmark",
     "authors": [
       "Ruizhong Qiu",
       "Weiliang Will Zeng",
       "James Ezick",
       "Christopher Lott",
       "Hanghang Tong"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations (ICLR 2025)",
     "arxiv_id": "2406.06647",
     "doi": "10.48550/arXiv.2406.06647"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "theoretical"],
   "key_findings": "LLMs fall far short of expert-level code efficiency: the strongest commercial model GPT-4 achieves only eff@1=0.454 despite pass@1=0.831 on the ENAMEL benchmark of 142 HumanEval problems with expert-written reference solutions. The efficiency gap stems from LLMs' inability to design advanced algorithms (eff@100=0.483 for ChatGPT on algorithm design subset) and near-total unawareness of implementation optimization. Encouraging efficiency via prompting barely helps, and even revealing the optimal algorithm in the prompt fails to elicit efficient code from LLMs.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states 'Our benchmark is publicly available at https://github.com/q-rz/enamel' in the abstract and provides a working GitHub URL."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark problems (from HumanEval/HumanEval+), expert reference solutions, and test case generators are released via the GitHub repository. Some generated code samples are re-used from the publicly available EvalPlus project."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "§C.1 specifies hardware ('8 NVIDIA A100 80GB GPUs', Google Cloud Ubuntu 20.04.6 LTS, Intel Xeon @ 2.20GHz, Python 3.10.12) but provides no requirements.txt, Dockerfile, or library version specifications sufficient to recreate the software environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the evaluation methodology in detail (§2-3, §C.1) and releases the benchmark, but does not include step-by-step reproduction instructions, scripts, or a 'Reproducing Results' section in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 3 report only point estimates for eff@k and pass@k. While Table 11 shows standard deviations for one model (Llama 3 70B Instruct) to demonstrate the Rao-Blackwellization benefit, no confidence intervals or error bars accompany the main reported results across 30 LLMs."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper ranks 30 LLMs and makes comparative claims (e.g., 'GPT-4 Turbo has higher eff@1 than GPT-4') but no statistical significance tests are applied to any comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports absolute eff@k and pass@k values with clear reference baselines (expert solutions = 1.000, HumanEval canonical = 0.455, HumanEval+ canonical = 0.513 in Table 2), allowing readers to assess effect magnitude. Differences between correctness and efficiency are reported numerically."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses 142 problems (explaining the exclusion of 22 trivial ones) and 100-200 code samples per problem, citing 'financial and computational constraints' (§C.1). No power analysis or formal sample size justification is provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The main results (Table 3) are single-point estimates. Table 11 shows standard deviations for one model to demonstrate the estimator benefit, but variance across experimental conditions or models is not systematically reported. Execution time variance is reduced via the Hodges-Lehmann estimator (R=6 repeats) but not reported to the reader."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against HumanEval and HumanEval+ canonical solutions (Table 2), and compares the ENAMEL benchmark against EffiBench and Mercury benchmarks (Table 9). The classic speedup metric is compared against eff@k (Table 7)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include contemporary benchmarks EffiBench (2024) and Mercury (2024), and the evaluation covers 30 LLMs including recent models like GPT-4 Turbo, Llama 3, Claude 3, and Mixtral (all 2024)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides ablation-style analyses: algorithm design vs implementation optimization subsets (Table 4/6), hyperparameter sensitivity (Table 10), random vs expert test generators (Table 8), different efficiency metrics (Table 7), and Rao-Blackwellization benefit (Table 11)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports both eff@k (efficiency) and pass@k (correctness) at multiple sample sizes k=1, 10, 100, under both greedy and sampling decoding."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is automated through execution time measurement and test case correctness. No human evaluation of code quality, benchmark validity, or LLM outputs is reported."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The LLMs are evaluated as pre-trained models with no fine-tuning on the ENAMEL benchmark. The expert reference solutions and strong test generators are newly created and were not available to the models during training."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4/6 provides breakdowns on algorithm design (20 problems) and implementation optimization (75 problems) subsets. Figure 2 shows per-problem difficulty distribution of passi@1 vs effi@1 across all 142 problems."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Tables 13-15 show detailed failure case studies where Llama 3 70B and Mixtral 8x22B fail to generate efficient algorithms even with Self-Refine prompting or explicit algorithm hints. Table 8 shows a case study where the Fermat primality test appears correct under random tests but fails under expert test cases."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 12 shows that encouraging efficiency via prompting 'can barely enhance the efficiency of generated code.' Tables 13-15 show that revealing the optimal algorithm in the prompt still fails to produce efficient code. These are explicit negative results."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: 'LLMs still fall short of generating expert-level efficient code' is supported by Table 3 (GPT-4 eff@1=0.454); 'LLMs struggle in designing advanced algorithms and are barely aware of implementation optimization' is supported by Tables 4/6. All specific numbers in the abstract match the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim — 'such deficiency is because current LLMs struggle in designing advanced algorithms' — is supported by controlled subset analysis (algorithm design vs implementation optimization subsets in Table 4) and the prompting experiments in §C.8 which rule out the alternative that prompting alone could address the issue."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Claims are generally bounded to 'under our benchmark ENAMEL.' Limitations (§D.2) explicitly acknowledge scope: 'This work considers standalone programming problems,' notes the inability to guarantee optimality of references, and identifies Python-only limitation. The title ('How Efficient is LLM-Generated Code?') is broader than the tested setting, but is framed as a question rather than a claim."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§C.8 tests and rules out the alternative explanation that prompting could fix efficiency (Self-Refine and algorithm-revealing prompts both fail). §C.5 discusses that LLMs have seen LeetCode solutions but not expert solutions. The paper considers that 'na¨ıve algorithms are easier to be generated correctly but are less efficient' (§4.1) as an explanatory factor."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures execution time under level-based evaluation and defines a formal eff@k metric. The measurement (actual code execution time normalized by expert reference) directly measures code efficiency, which is the claimed outcome. No proxy gap exists between measurement and claim."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "§C.1 specifies exact model versions: 'claude-3-opus-20240229', 'claude-3-sonnet-20240229', 'claude-3-haiku-20240307', 'gpt-4-1106-preview', 'gpt-4-0613'. Open-source models are identified by full names (e.g., 'Llama 3 70B Instruct')."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The prompts are the standard HumanEval function signatures and docstrings, which are publicly available. The paper explicitly references HumanEval (Chen et al., 2021) and HumanEval+ (Liu et al., 2023a) as prompt sources. Example prompts are shown (e.g., Fig. 1). The benchmark GitHub repository contains the full prompt set."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "§C.1 reports: temperature 0.8, top_p 0.95 for sampling; α=2, R=6, h1=h2=3, h3=4, M0=8, M1=M2=M3=4 for evaluation. Table 10 provides sensitivity analysis for α, h1, h2, h3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct code generation from prompts without any multi-turn, tool-use, or agent workflow."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "§3.1 documents problem selection: 164 HumanEval problems → exclude 22 with Θ(1) time complexity → 142 problems. §C.1 describes code generation (re-use from EvalPlus for some models, self-generated for others). The evaluation pipeline (levels 0-3, correctness filtering, efficiency scoring) is documented in §2."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "§D 'Concluding Remarks' includes §D.1 'Scalability of Benchmark Development' and §D.2 'Other Limitations & Future Work' as dedicated subsections with substantive discussion of multiple limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "§D.2 identifies specific threats: standalone problems only (not complex software development), inability to guarantee optimality of reference solutions ('the efficiency score can be greater than 1'), no space efficiency evaluation, no advanced prompting techniques, and difficulty of automatic time complexity measurement."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "§D.2 explicitly states: 'This work considers standalone programming problems,' 'this work focuses on benchmarking code efficiency without more advanced prompting techniques,' and notes specific things not tested (space efficiency, multi-file dependencies, automatic complexity measurement)."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark (problems, reference solutions, test generators) is released at the GitHub URL. However, the raw experimental data — generated code samples from 30 LLMs and their per-problem execution times — is not explicitly stated to be released. Partial re-use of EvalPlus code samples is mentioned but full raw data for independent verification of reported numbers is not confirmed."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "§C.1 describes code collection: re-use from EvalPlus for some models, self-generation with temperature 0.8, top_p 0.95, 100-200 samples per problem. Evaluation on Google Cloud VMs with specific hardware. §3 describes reference solution and test generator creation by a human expert."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The data sources are standard LLM APIs and the publicly available HumanEval/HumanEval+ benchmark."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is fully documented: problem selection (§3.1), reference solution creation (§3.2), test generator curation (§3.3), code generation (§C.1), level-based evaluation with correctness filtering (§2.1), efficiency scoring (§2.2), and metric estimation via Rao-Blackwellization (§2.3, Algorithm 1)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The acknowledgments section states: 'This work is supported by NSF (2134079), and NIFA (2020-67021-32799), and IBM-Illinois Discovery Accelerator Institute.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: University of Illinois Urbana-Champaign (Qiu, Tong) and Qualcomm AI Research (Zeng, Ezick, Lott). The footnote notes 'Qualcomm AI Research is an initiative of Qualcomm Technologies, Inc.'"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funders (NSF, NIFA, IBM-Illinois) are government agencies or an academic institute with no direct financial stake in which LLM ranks highest in code efficiency. The paper evaluates third-party models (GPT-4, Claude 3, Llama 3, etc.) rather than products of the funders."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Authors from Qualcomm AI Research may have interests related to code efficiency evaluation, but no disclosure statement addresses this."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the 30 evaluated models. This is relevant because HumanEval (published 2021) could be in the training data of post-2021 models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "§C.5 and §D.1 explicitly discuss this: 'LLMs (i) have seen the public solutions on LeetCode (editorials and community solutions) but (ii) have never seen our expert-written efficient solutions.' They acknowledge HumanEval solutions are in training data and argue their expert solutions provide novel reference points."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§D.1 discusses contamination directly: 'We did not use problems or solutions from online judges because their public solutions are already in LLMs' pretraining corpuses.' They argue that while HumanEval problems may be contaminated for correctness, their novel expert solutions and test generators mitigate contamination for efficiency evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLMs on a code efficiency benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The 'human expert' who wrote reference solutions and test generators is an author, not a research subject."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token consumption, or per-example inference costs are reported. The paper mentions using Google Cloud VMs and A100 GPUs but does not quantify inference costs."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions hardware (8 NVIDIA A100 80GB GPUs for generation, Google Cloud VMs for evaluation) but does not state total GPU hours, API spend, or overall computational budget."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. While the Rao-Blackwellized estimator reduces variance from sampling, the paper does not report results across multiple random seeds or assess seed sensitivity."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "§C.1 explicitly states: R=6 repeats per test case for execution time estimation, 100-200 code samples per problem depending on model size, and greedy vs sampling decoding modes."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses fixed default hyperparameters (α=2, h1=3, h2=3, h3=4) without search, and provides sensitivity analysis in Table 10. LLM generation uses standard settings (temperature 0.8, top_p 0.95). No tuning was performed, so no search budget applies."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 30 models' results are reported comprehensively in Table 3 under fixed default settings. No cherry-picking — results for all models under both greedy and sampling modes are shown. Hyperparameters are fixed defaults, not selected to favor any outcome."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No formal statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their ENAMEL benchmark against EffiBench and Mercury (Table 9), claiming ENAMEL is 'more challenging,' but do not acknowledge potential bias in this self-comparison. They also evaluate their expert solutions against HumanEval+ canonical solutions without discussing author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper compares models ranging from 1B to 70B parameters alongside commercial models with undisclosed sizes, without normalizing for compute budget or discussing the compute-performance tradeoff."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "§2 extensively discusses construct validity: §2.1 explains why level-based evaluation differentiates algorithms, §2.2 addresses right-censored execution time, §C.3 compares eff@k vs speedup metrics. §C.4 demonstrates that their expert test generators correctly identify wrong-but-efficient code (Table 8). §C.5 compares difficulty against other benchmarks."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved — direct code generation from prompts without any agentic workflow."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "§C.5 and §D.1 discuss that HumanEval problems and LeetCode solutions existed before model training. They argue their expert solutions are novel ('have never seen our expert-written efficient solutions'), partially mitigating temporal leakage for efficiency evaluation."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the evaluation setup could leak answer information through the prompts or context. HumanEval docstrings sometimes contain hints about solution approaches, which is not addressed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the 142 HumanEval problems are independent of each other or whether similar problems in training data could affect results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, n-gram overlap, membership inference) are applied. The argument that expert solutions are novel is conceptual rather than empirically verified."
       }
     }
   },
   "claims": [
     {
       "claim": "LLMs still fall short of generating expert-level efficient code: even the strongest commercial LLM GPT-4 has eff@1=0.454 despite pass@1=0.831.",
       "evidence": "Table 3 shows eff@k and pass@k for 30 LLMs. GPT-4 greedy: eff@1=0.454, pass@1=0.831. GPT-4 Turbo greedy: eff@1=0.470, pass@1=0.796. Most LLMs cannot reach eff@1>0.3.",
       "supported": "strong"
     },
     {
       "claim": "eff@k is consistently much lower than pass@k across all LLMs, model sizes, and sample sizes k.",
       "evidence": "Table 3 systematically shows this gap across all 30 models. For example, Llama 3 70B Instruct: eff@1=0.438 vs pass@1=0.747; Claude 3 Sonnet: eff@1=0.365 vs pass@1=0.677.",
       "supported": "strong"
     },
     {
       "claim": "LLMs struggle in designing advanced algorithms: even ChatGPT has eff@100=0.483 on the algorithm design subset.",
       "evidence": "Table 4/6 shows results on 20 hard algorithm design problems. ChatGPT eff@100=0.483, Llama 3 70B eff@100=0.359. All models have low eff@1 on this subset.",
       "supported": "moderate"
     },
     {
       "claim": "LLMs are barely aware of implementation optimization: improvement with larger sample sizes is mainly from random sampling generating diverse implementations.",
       "evidence": "Table 4/6 on 75 implementation optimization problems. Phind Code Llama V2: eff@1=0.351 but eff@100=0.732. The large improvement from k=1 to k=100 suggests random diversity rather than intentional optimization.",
       "supported": "moderate"
     },
     {
       "claim": "Encouraging LLMs to generate 'the most efficient algorithm' via prompting barely enhances code efficiency.",
       "evidence": "Table 12 shows Llama 3 70B eff@1 drops from 0.421 to 0.418 with efficiency-encouraging prompt; Mixtral 8x22B increases only from 0.408 to 0.426. Tested on only 2 LLMs with greedy decoding.",
       "supported": "moderate"
     },
     {
       "claim": "Many canonical solutions in HumanEval and HumanEval+ are wrong or exceed the time limit under strong test cases.",
       "evidence": "Table 2: 11 HumanEval and 4 HumanEval+ canonical solutions found wrong; 34 HumanEval and 27 HumanEval+ exceed time limits. Specific examples in Table 1 and Appendix E.",
       "supported": "strong"
     },
     {
       "claim": "The Rao-Blackwellized eff@k estimator is unbiased and achieves lower variance than the vanilla estimator.",
       "evidence": "Theorem 1 (§2.3, proof in §B) provides formal proofs of unbiasedness and variance reduction. Table 11 empirically confirms: vanilla std=0.20 vs Rao-Blackwellized std=0.02 for k=1.",
       "supported": "strong"
     },
     {
       "claim": "The classic speedup metric overestimates efficiency due to right-censored execution time, producing unreasonable rankings.",
       "evidence": "Table 7 shows that under speedup, Mixtral 8x22B and Llama 3 70B appear to outperform GPT-4, while eff@1 gives a more reasonable ranking with GPT-4 Turbo and GPT-4 at top.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "Single unnamed human expert",
       "detail": "All reference solutions and test case generators were created by a single 'human expert' who is likely one of the authors. There is no inter-rater reliability check, no second expert validation, and the expert's identity is not disclosed. The entire benchmark's quality standard depends on this one person's judgment."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Rankings of 30 LLMs and comparative claims (e.g., 'GPT-4 Turbo has higher eff@1 than GPT-4') are based solely on point estimates without any significance testing. Given the variance in code generation, some ranking differences may not be meaningful."
     },
     {
       "flag": "Uneven evaluation across models",
       "detail": "Some models are evaluated only with greedy decoding (GPT-4, GPT-4 Turbo, Claude 3 Opus) while others have sampling results with 100-200 samples. The most expensive commercial models have the least thorough evaluation, making comparisons at higher k impossible."
     },
     {
       "flag": "Narrow domain generalization",
       "detail": "All 142 problems are standalone Python functions from HumanEval — a single language, single-function granularity. The title 'How Efficient is LLM-Generated Code?' implies broader applicability than this narrow domain supports. Real-world efficiency involves multi-file, multi-language, system-level optimization."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric, which ENAMEL extends to efficiency evaluation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus/HumanEval+ with stronger test cases; ENAMEL re-uses their code samples and extends their problem set for efficiency."
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "authors": ["Dong Huang", "Jie M. Zhang", "Yuhao Qing", "Heming Cui"],
       "year": 2024,
       "arxiv_id": "2402.02037",
       "relevance": "Contemporary code efficiency benchmark that ENAMEL compares against; uses GPT-generated test cases rather than expert-written ones."
     },
     {
       "title": "Mercury: An efficiency benchmark for LLM code synthesis",
       "authors": ["Mingzhe Du", "Anh Tuan Luu", "Bin Ji", "See-Kiong Ng"],
       "year": 2024,
       "arxiv_id": "2402.07844",
       "relevance": "Another code efficiency benchmark using LeetCode problems; ENAMEL compares against it and argues for expert-written rather than crowd-sourced solutions."
     },
     {
       "title": "On evaluating the efficiency of source code generated by LLMs",
       "authors": ["Changan Niu", "Ting Zhang", "Chuanyi Li", "Bin Luo", "Vincent Ng"],
       "year": 2024,
       "relevance": "Early work on LLM code efficiency evaluation; ENAMEL addresses its limitations including use of only one code sample and reliance on existing HumanEval test cases."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP code generation benchmark; discussed as a correctness-focused benchmark that overlooks efficiency."
     },
     {
       "title": "Measuring coding challenge competence with APPS",
       "authors": ["Dan Hendrycks", "Steven Basart", "Saurav Kadavath"],
       "year": 2021,
       "relevance": "Introduces APPS benchmark for code generation; cited as another correctness-focused evaluation framework."
     },
     {
       "title": "Competition-level code generation with AlphaCode",
       "authors": ["Yujia Li", "David Choi", "Junyoung Chung"],
       "year": 2022,
       "relevance": "DeepMind's competition-level code generation system; relevant as a major code LLM trained on online judge data, raising contamination concerns."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Roziere", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Major open-source code LLM family evaluated in the benchmark; multiple sizes (7B-70B) provide model scaling analysis."
     },
     {
       "title": "Self-Refine: Iterative refinement with self-feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2024,
       "relevance": "Prompting technique tested in §C.8 for improving code efficiency; shown to be ineffective for generating efficient algorithms."
     },
     {
       "title": "Can large language models write parallel code?",
       "authors": ["Daniel Nichols", "Joshua H. Davis", "Zhaojun Xie"],
       "year": 2024,
       "relevance": "Evaluates LLM-generated parallel code efficiency; one of the first works on code efficiency beyond correctness."
     },
     {
       "title": "DevBench: A comprehensive benchmark for software development",
       "authors": ["Bowen Li", "Wenhan Wu", "Ziwei Tang"],
       "year": 2024,
       "arxiv_id": "2403.08604",
       "relevance": "Referenced as a potential extension target for generalizing ENAMEL's methodology to complex multi-file software development."
     },
     {
       "title": "MultiPL-E: A scalable and extensible approach to benchmarking neural code generation",
       "authors": ["Federico Cassano", "John Gouwar", "Daniel Nguyen"],
       "year": 2022,
       "arxiv_id": "2208.08227",
       "relevance": "Multilingual code generation benchmark; cited as a correctness-focused framework that does not address efficiency."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The ENAMEL benchmark is a usable tool for evaluating LLM code efficiency, relevant to LLM developers and benchmark practitioners, but not directly applicable to day-to-day software engineering."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that GPT-4 achieves only eff@1=0.454 despite pass@1=0.831 challenges the assumption that high correctness implies high-quality code, revealing a significant blind spot in current evaluation."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No safety, security, or risk implications; the paper is about code efficiency evaluation."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild 'existing benchmarks are inadequate' angle — 11 HumanEval canonical solutions are wrong and 34 exceed time limits under expert tests — but framed constructively rather than confrontationally."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The benchmark is publicly available on GitHub (github.com/q-rz/enamel), though running the full evaluation requires significant compute resources."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Published at ICLR 2025, evaluates well-known models (GPT-4, Claude 3, Llama 3), with Qualcomm AI Research and UIUC affiliations."
     }
   }
 }