ai-research-survey

scan-v5.json (28155B)

---

 {
   "scan_version": 5,
   "paper_type": "empirical",
   "paper": {
     "title": "How Efficient is LLM-Generated Code? A Rigorous & High-Standard Benchmark",
     "authors": [
       "Ruizhong Qiu",
       "Weiliang Will Zeng",
       "James Ezick",
       "Christopher Lott",
       "Hanghang Tong"
     ],
     "year": 2024,
     "venue": "International Conference on Learning Representations",
     "arxiv_id": "2406.06647",
     "doi": "10.48550/arXiv.2406.06647"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are backed by the paper: the eff@k metric is formally derived (§2), Rao-Blackwellization is proven in Theorem 1, expert reference solutions are described in §3.2, and evaluation of 30 LLMs with GPT-4 eff@1=0.454 is shown in Table 3.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims LLMs underperform 'because' they struggle with algorithm design and implementation optimization, but this is inferred from subset performance differences rather than controlled experiments isolating the causal mechanism.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The paper scopes conclusions to 142 HumanEval-derived Python problems and the 30 evaluated LLMs; limitations in Appendix D explicitly note the benchmark covers standalone problems, not complex multi-file software development.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper explores prompting as an alternative explanation (Appendix C.8) but does not seriously consider whether benchmark design choices (problem set, time limits, single expert) could explain observed LLM deficiencies.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly distinguishes eff@k (execution time efficiency against expert reference) from pass@k (functional correctness) and explicitly discusses what the metric measures vs. what is claimed, including limitations of measuring only time efficiency.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix D.2 provides a dedicated 'Other Limitations & Future Work' section listing multiple specific limitations including standalone problem scope, unverifiable reference optimality, time-only efficiency focus, and lack of advanced prompting techniques.",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are named: reference solutions may not be provably optimal (efficiency score can exceed 1.0), only standalone problems are studied (not real-world multi-file code), and single expert involvement is acknowledged as a scalability concern.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly states it does NOT address space efficiency, complex software projects, or automatic time complexity measurement, and Appendix D.1 explains why crowd-sourcing and online judge solutions were excluded.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments disclose NSF (2134079), NIFA (2020-67021-32799), and IBM-Illinois Discovery Accelerator Institute as funding sources.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with UIUC and Qualcomm AI Research are disclosed on the title page; the Qualcomm footnote clarifies that datasets were evaluated at UIUC.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "NSF and NIFA are government agencies independent of LLM outcomes; IBM funds the lab but does not make models evaluated in the paper; Qualcomm authors work on the benchmark methodology, not on evaluated models.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are formally defined: eff@k is defined in §2.3 (Eq. 5-6), 'right-censored execution time' is explained with reference to statistics literature, and 'algorithm design' vs. 'implementation optimization' are distinguished with concrete examples.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "The introduction lists four explicit contributions: (1) eff@k metric with unbiased estimator, (2) expert reference solutions, (3) strong test case generators, and (4) evaluation of 30 LLMs.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "The paper identifies five specific methodological gaps (C1-C5) in prior work (Niu et al., Huang et al., Du et al.) and shows empirically how prior metrics fail (Table 7 speedup comparison, Table 8 random test case comparison).",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "empirical": {
       "artifacts": {
         "code_released": {
           "applies": true,
           "answer": true,
           "justification": "The abstract explicitly states 'Our benchmark is publicly available at https://github.com/q-rz/enamel.'",
           "source": "haiku"
         },
         "data_released": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark problems (derived from HumanEval/HumanEval+), expert reference solutions, and test case generators are released on GitHub; base HumanEval and HumanEval+ are standard public benchmarks.",
           "source": "haiku"
         },
         "environment_specified": {
           "applies": true,
           "answer": false,
           "justification": "Appendix C.1 mentions Ubuntu 20.04.6 LTS, Intel Xeon CPU @ 2.20GHz, Python 3.10.12, and 8 NVIDIA A100 80GB GPUs, but no requirements.txt, Dockerfile, or formal dependency specification is provided.",
           "source": "haiku"
         },
         "reproduction_instructions": {
           "applies": true,
           "answer": false,
           "justification": "The paper does not include step-by-step instructions for reproducing results; the GitHub repository may contain them but they are not described in the paper text.",
           "source": "haiku"
         }
       },
       "statistical_methodology": {
         "confidence_intervals_or_error_bars": {
           "applies": true,
           "answer": false,
           "justification": "Main results tables (Table 3, Table 4) report only point estimates for eff@k and pass@k without confidence intervals or error bars; Table 11 reports standard deviation only for the estimator comparison, not for LLM comparisons.",
           "source": "haiku"
         },
         "significance_tests": {
           "applies": true,
           "answer": false,
           "justification": "No statistical significance tests are applied to comparative claims between LLMs; rankings and performance differences are presented without p-values or hypothesis tests.",
           "source": "haiku"
         },
         "effect_sizes_reported": {
           "applies": true,
           "answer": true,
           "justification": "The eff@k values themselves serve as effect sizes against the expert reference (1.000); GPT-4 Turbo achieving eff@1=0.470 vs expert level quantifies the gap explicitly.",
           "source": "haiku"
         },
         "sample_size_justified": {
           "applies": true,
           "answer": false,
           "justification": "The 142 problems are explained by problem selection criteria (excluding O(1) trivial problems) and the 100-200 code samples per problem by computational constraints, but no statistical power analysis justifies these numbers.",
           "source": "haiku"
         },
         "variance_reported": {
           "applies": true,
           "answer": false,
           "justification": "Standard deviations are reported only for the estimator comparison (Table 11); no variance across runs or problems is reported for the main LLM evaluation results.",
           "source": "haiku"
         }
       },
       "evaluation_design": {
         "baselines_included": {
           "applies": true,
           "answer": true,
           "justification": "Table 2 compares ENAMEL against HumanEval and HumanEval+ canonical solutions showing eff@1=0.455 and 0.513 respectively vs. 1.000 for ENAMEL; Table 9 compares against EffiBench and Mercury.",
           "source": "haiku"
         },
         "baselines_contemporary": {
           "applies": true,
           "answer": true,
           "justification": "Compared LLMs include GPT-4 Turbo, Claude 3 Opus/Sonnet/Haiku, and Llama 3 (all 2024 models); efficiency benchmark comparisons include EffiBench and Mercury (both 2024).",
           "source": "haiku"
         },
         "ablation_study": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.6 ablates hyperparameters (timeout factor α, level hardnesses h1, h2, h3); C.3 compares eff@k vs. the speedup metric; C.4 compares expert vs. random test case generators.",
           "source": "haiku"
         },
         "multiple_metrics": {
           "applies": true,
           "answer": true,
           "justification": "Both eff@k and pass@k are reported at multiple sample sizes k=1, 10, 100 for all models, providing both correctness and efficiency perspectives.",
           "source": "haiku"
         },
         "human_evaluation": {
           "applies": false,
           "answer": false,
           "justification": "Human evaluation of LLM outputs is not applicable; the paper uses automated execution-time-based evaluation against expert reference solutions.",
           "source": "haiku"
         },
         "held_out_test_set": {
           "applies": false,
           "answer": false,
           "justification": "This is a benchmark evaluation study, not a prediction task with train/test splits; the fixed problem set is the evaluation target.",
           "source": "haiku"
         },
         "per_category_breakdown": {
           "applies": true,
           "answer": true,
           "justification": "Table 4 provides separate results for an Algorithm Design Subset (20 hard problems) and an Implementation Optimization Subset (75 problems); Figure 2 shows per-problem difficulty distribution.",
           "source": "haiku"
         },
         "failure_cases_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.8 provides detailed case studies of LLM failures on problem #36, showing that both Self-Refine and explicit algorithm hints fail to elicit O(log n) solutions from Llama 3 and Mixtral.",
           "source": "haiku"
         },
         "negative_results_reported": {
           "applies": true,
           "answer": true,
           "justification": "Table 12 shows that encouraging LLMs to generate 'the most efficient algorithm' barely changes eff@1 (e.g., Llama 3 70B drops from 0.421 to 0.418), and Appendix C.8 demonstrates Self-Refine fails to improve efficiency.",
           "source": "haiku"
         }
       },
       "setup_transparency": {
         "model_versions_specified": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.1 lists exact API snapshot IDs: claude-3-opus-20240229, claude-3-sonnet-20240229, claude-3-haiku-20240307, gpt-4-1106-preview, gpt-4-0613.",
           "source": "haiku"
         },
         "prompts_provided": {
           "applies": true,
           "answer": false,
           "justification": "The paper states prompts from HumanEval+ are reused but does not reproduce the actual prompt text; only the prompt engineering variants in Appendix C.8 are shown, not the baseline prompts.",
           "source": "haiku"
         },
         "hyperparameters_reported": {
           "applies": true,
           "answer": true,
           "justification": "Appendix C.1 reports all evaluation hyperparameters: α=2, R=6, h1=h2=3, h3=4, M0=8, M1=M2=M3=4, temperature=0.8, top_p=0.95, and sample sizes (100 or 200 per problem).",
           "source": "haiku"
         },
         "scaffolding_described": {
           "applies": false,
           "answer": false,
           "justification": "No agentic scaffolding is used; models generate code directly from problem prompts without multi-step reasoning pipelines.",
           "source": "haiku"
         },
         "data_preprocessing_documented": {
           "applies": true,
           "answer": true,
           "justification": "Section 3.1 documents the problem selection process: starting from 164 HumanEval problems, excluding 22 with Θ(1) complexity for two documented reasons, yielding 142 problems.",
           "source": "haiku"
         }
       },
       "data_integrity": {
         "raw_data_available": {
           "applies": true,
           "answer": true,
           "justification": "The benchmark (problems, reference solutions, test case generators) is publicly available at GitHub; the benchmark constitutes the primary data artifact.",
           "source": "haiku"
         },
         "data_collection_described": {
           "applies": true,
           "answer": true,
           "justification": "Section 3 describes benchmark development in detail: problem selection criteria (§3.1), human expert algorithm design and implementation optimization process (§3.2), and test case generator curation methodology (§3.3).",
           "source": "haiku"
         },
         "recruitment_methods_described": {
           "applies": false,
           "answer": false,
           "justification": "No participant recruitment is involved; the 'human expert' is a benchmark developer, not a study participant.",
           "source": "haiku"
         },
         "data_pipeline_documented": {
           "applies": true,
           "answer": true,
           "justification": "The full evaluation pipeline is documented: code generation (temperature, sample size, model API), level-based execution (test cases, time limits, scoring formula), and eff@k computation (Algorithm 1).",
           "source": "haiku"
         }
       },
       "contamination": {
         "training_cutoff_stated": {
           "applies": true,
           "answer": false,
           "justification": "Training data cutoffs for evaluated models are not stated; the paper treats models as black boxes and does not report when their training data was collected.",
           "source": "haiku"
         },
         "train_test_overlap_discussed": {
           "applies": true,
           "answer": true,
           "justification": "Appendix D.1 explicitly discusses contamination: online judge solutions were excluded because 'their public solutions are already in LLMs' pretraining corpuses,' and expert-written solutions are novel.",
           "source": "haiku"
         },
         "benchmark_contamination_addressed": {
           "applies": true,
           "answer": true,
           "justification": "Table 9 notes that ENAMEL is harder than EffiBench/Mercury because 'LLMs have seen the public solutions on LeetCode but have never seen our expert-written efficient solutions'; this directly addresses contamination.",
           "source": "haiku"
         }
       },
       "human_studies": {
         "pre_registered": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "irb_or_ethics_approval": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "demographics_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "inclusion_exclusion_criteria": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "randomization_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "blinding_described": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         },
         "attrition_reported": {
           "applies": false,
           "answer": false,
           "justification": "No human participants.",
           "source": "haiku"
         }
       },
       "cost_and_practicality": {
         "inference_cost_reported": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions financial constraints motivated sampling size choices but does not report actual inference costs or latency figures for any model.",
           "source": "haiku"
         },
         "compute_budget_stated": {
           "applies": true,
           "answer": false,
           "justification": "Hardware (8 NVIDIA A100 80GB GPUs, Google Cloud VMs) is mentioned but no total compute budget (GPU-hours, cost) is stated.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "Most existing code generation benchmarks focus on functional correctness and overlook efficiency.",
       "evidence": "Related work section and C1-C5 challenge framing document that HumanEval, MBPP, APPS, EvalPlus, and others measure only correctness; only a few recent benchmarks (EffiBench, Mercury) address efficiency.",
       "supported": "strong"
     },
     {
       "claim": "The eff@k estimator is unbiased and has provably lower variance than naive estimation.",
       "evidence": "Theorem 1 provides formal proof of unbiasedness and variance reduction via Rao-Blackwellization; Table 11 empirically confirms standard deviation drops from 0.20 to 0.02 at k=1.",
       "supported": "strong"
     },
     {
       "claim": "LLMs fall far short of expert-level efficient code; even GPT-4 achieves only eff@1=0.454 against expert reference (1.000).",
       "evidence": "Table 3 shows eff@k for 30 LLMs; GPT-4 Turbo tops at eff@1=0.470, with most models below 0.3; eff@k is consistently much lower than pass@k across all models.",
       "supported": "strong"
     },
     {
       "claim": "LLMs that generate correct code are not necessarily equally good at efficient code (pass@k rank ≠ eff@k rank).",
       "evidence": "Table 3 shows GPT-4 Turbo has higher eff@1=0.470 but lower pass@1=0.796 than GPT-4 (pass@1=0.831, eff@1=0.454); Table 7 shows speedup metric inverts rankings further.",
       "supported": "strong"
     },
     {
       "claim": "LLMs struggle in designing advanced algorithms; ChatGPT achieves only eff@100=0.483 on the 20-problem algorithm design subset.",
       "evidence": "Table 4 shows all models have substantially lower eff@k on the Algorithm Design Subset; case studies in Appendix C.8 show models fail even when the efficient algorithm is explicitly revealed.",
       "supported": "strong"
     },
     {
       "claim": "Prompt engineering (encouraging efficiency or Self-Refine) cannot significantly improve code efficiency.",
       "evidence": "Table 12 shows Llama 3 70B drops from eff@1=0.421 to 0.418 with efficiency prompting; Table 13 shows both LLMs remain at O(n) after Self-Refine on problem #36.",
       "supported": "moderate"
     },
     {
       "claim": "HumanEval and HumanEval+ canonical solutions are themselves inefficient by expert standards (eff@1=0.455 and 0.513 respectively).",
       "evidence": "Table 2 directly compares canonical solution performance under ENAMEL evaluation; Table 1 shows several canonical solutions have sub-optimal time complexity.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "ENAMEL, a new benchmark of 142 Python problems with expert-written optimal reference solutions and strong test case generators, reveals that all 30 evaluated LLMs fall dramatically short of expert-level code efficiency. The strongest commercial model (GPT-4 Turbo) achieves only eff@1=0.470 against expert reference, with the gap widening specifically on algorithm design problems (ChatGPT eff@100=0.483 on a hard-problem subset). A key methodological contribution is the eff@k metric, which properly handles right-censored execution times via a Rao-Blackwellized estimator with provably lower variance, correcting a systematic overestimation flaw in prior efficiency metrics. Prompt engineering (including encouraging efficiency and Self-Refine) provides negligible improvement, suggesting the deficiency reflects fundamental capability limitations rather than prompting artifacts.",
   "red_flags": [
     {
       "flag": "Single-expert benchmark construction",
       "detail": "All reference solutions and test case generators were created by a single human expert, introducing potential idiosyncratic bias in what constitutes 'optimal' and 'strong' test cases; inter-rater reliability is not assessed."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims between LLMs (e.g., rankings in Table 3, Table 7) are based on point estimates without confidence intervals or significance tests, making it impossible to assess whether performance differences are meaningful."
     },
     {
       "flag": "HumanEval problem contamination not fully addressed",
       "detail": "While expert solutions are novel, the HumanEval problem descriptions themselves have been widely published since 2021 and are almost certainly in LLM training data, potentially confounding the evaluation of problem comprehension vs. algorithmic capability."
     },
     {
       "flag": "Unverifiable optimality of reference solutions",
       "detail": "The paper acknowledges that reference solutions may not be provably optimal (efficiency scores can theoretically exceed 1.0), meaning the expert standard is aspirational rather than provably tight."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code (Codex/HumanEval)",
       "relevance": "Primary benchmark whose problems ENAMEL extends; introduces the pass@k metric that eff@k generalizes"
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation (EvalPlus/HumanEval+)",
       "relevance": "Basis for ENAMEL's problem set and stronger test cases; showed original HumanEval tests were insufficient"
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "relevance": "Prior efficiency benchmark compared against ENAMEL; uses random GPT-generated test cases (challenge C5)"
     },
     {
       "title": "Mercury: An efficiency benchmark for LLM code synthesis",
       "relevance": "Another efficiency benchmark compared against ENAMEL; uses LeetCode problems with contamination concerns"
     },
     {
       "title": "On evaluating the efficiency of source code generated by LLMs (FORGE 2024)",
       "relevance": "Prior efficiency evaluation work; uses single code sample and uncensored execution time (challenges C1, C2)"
     },
     {
       "title": "Self-Refine: Iterative refinement with self-feedback",
       "relevance": "Prompting technique tested in Appendix C.8 as a potential approach to improve code efficiency"
     },
     {
       "title": "Can large language models write parallel code?",
       "relevance": "Related work on LLM code efficiency evaluation in a different domain (parallelism)"
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "relevance": "One of the major open-source code LLMs evaluated in the benchmark study"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "The benchmark is publicly released and directly applicable to developers evaluating LLMs for performance-critical code generation tasks."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that correctness rank does not predict efficiency rank (GPT-4 Turbo beats GPT-4 on efficiency despite lower correctness) challenges the common assumption that better models are uniformly better."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper raises no AI safety or risk concerns; it is a pure capability evaluation benchmark."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper implicitly criticizes prior efficiency benchmarks (EffiBench, Mercury) for methodological flaws and shows canonical HumanEval solutions are themselves inefficient, creating mild controversy."
     },
     "demo_ability": {
       "score": 2,
       "justification": "The benchmark is available on GitHub and can be used immediately to evaluate any LLM on code efficiency."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "UIUC and Qualcomm AI Research are known institutions but not the dominant brands in LLM evaluation; ICLR venue adds credibility."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "36371376",
         "title": "Starlink v2 reached the brightness reduction target recommended by astronomers",
         "points": 122,
         "comments": 76,
         "url": "https://news.ycombinator.com/item?id=36371376"
       },
       {
         "hn_id": "40739778",
         "title": "RAR-B: Reasoning as Retrieval Benchmark",
         "points": 11,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40739778"
       },
       {
         "hn_id": "40795244",
         "title": "How Far Can Transformers Reason? The Locality Barrier and Inductive Scratchpad",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40795244"
       },
       {
         "hn_id": "40043146",
         "title": "Why do small language models underperform?",
         "points": 4,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40043146"
       },
       {
         "hn_id": "40651110",
         "title": "Google: Towards a Personal Health Large Language Model",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40651110"
       },
       {
         "hn_id": "40341096",
         "title": "Linearizing Large Language Models",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40341096"
       },
       {
         "hn_id": "39132737",
         "title": "The \"Pac-Man'' Gripper: Tactile Sensing and Grasping Through Thin-Shell Buckling",
         "points": 2,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=39132737"
       },
       {
         "hn_id": "23526567",
         "title": "Training GANs just became *a lot* easier",
         "points": 1,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=23526567"
       }
     ],
     "top_points": 122,
     "total_points": 151,
     "total_comments": 78
   }
 }