ai-research-survey

scan-v5.json (21069B)

---

 {
   "scan_version": 5,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluating Language Models for Efficient Code Generation",
     "authors": [
       "Jiawei Liu",
       "Songrun Xie",
       "Junhao Wang",
       "Yuxiang Wei",
       "Yifeng Ding",
       "Lingming Zhang"
     ],
     "year": 2024,
     "venue": "COLM 2024",
     "arxiv_id": "2408.06450",
     "doi": "10.48550/arXiv.2408.06450"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "All abstract claims are substantiated: the 121-task EVALPERF is built and evaluated, scaling law failure is demonstrated in Figure 5, and instruction tuning efficiency gains are shown in Figure 4.",
         "source": "haiku"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'general instruction tuning benefits code efficiency' are supported only by observational model comparisons (base vs. instruct variants), not controlled experiments; confounders such as training data quality are not ruled out.",
         "source": "haiku"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The paper draws broad conclusions ('scaling law fails for code efficiency') based on one language (Python) and tasks from HumanEval+/MBPP+ only, without bounding claims to these specific settings.",
         "source": "haiku"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are considered for why instruction tuning improves efficiency or why scaling fails; the paper presents a single interpretation without considering competing hypotheses.",
         "source": "haiku"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes instruction count (#instructions) from physical runtime, justifies the proxy in Appendix A.2, and separately defines DPS vs. DPSnorm to capture different aspects of performance.",
         "source": "haiku"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations or threats-to-validity section; only a brief mention of future extensions in the conclusion ('our future efforts will continuously extend EVALPERF').",
         "source": "haiku"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats are articulated; cross-platform variation is addressed empirically but framed as a positive result, not a threat, and benchmark coverage and contamination risk are not discussed.",
         "source": "haiku"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what EVALPERF results do NOT show; Python-only coverage and reliance on HumanEval/MBPP source tasks are not framed as scope limitations.",
         "source": "haiku"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Funding is disclosed in the Acknowledgment: NSF grant CCF-2131943, Kwai Inc, and OpenAI Researcher Access Program API credits.",
         "source": "haiku"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Illinois Urbana-Champaign and Tongji University, with email contact provided.",
         "source": "haiku"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Neither Kwai Inc nor NSF produce the evaluated LLMs (CodeLlama, DeepSeekCoder, StarCoder, GPT-4); no evaluated model is linked to the funders, so there is no direct financial stake in the outcome.",
         "source": "haiku"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is provided beyond the funding acknowledgment; patents, equity, or consulting arrangements are not declared.",
         "source": "haiku"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined: 'code efficiency' is operationalized as instruction count, 'DPS' and 'DPSnorm' are formally defined with equations, and 'performance-exercising' criteria are specified numerically.",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four explicit contributions are enumerated in the introduction: a new evaluation dimension, the DPE technique, the EVALPERF benchmark, and an empirical study.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 compares DPE against HumanEval, MBPP, EvalPlus, PIE, and contemporaneous sibling benchmarks (EffiBench, ECCO), explaining specific shortcomings of each that DPE addresses.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly argues in Section 1 that existing benchmarks fail due to light computation and inadequate metrics, and that performance-exercising inputs with a reference-relative compound metric are necessary for valid efficiency measurement.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": false,
           "justification": "The paper requires at least 4 performance clusters per task but does not characterize the distribution of task difficulty across EVALPERF (no histogram, no easy/medium/hard tiers reported).",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": true,
           "justification": "Filtering criteria explicitly guard against floor effects (>10k instruction minimum) and ensure performance diversity (≥4 clusters required); ceiling effects are mitigated by requiring the strongest input within 20-second / 16GB limits.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human performance baseline is provided; the paper evaluates only LLMs and uses LLM-generated solutions as reference clusters.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "DPS is formally defined with a mathematical formula, Appendix A.2 extensively justifies the choice over relative speedup and physical runtime, and edge cases (e.g., solution slower than all references → score 0) are addressed.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "Tasks are drawn directly from HumanEval and MBPP, which are public benchmarks almost certainly in LLM training data; no temporal splits, canary strings, or anti-gaming measures are discussed.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper only briefly notes 'our future efforts will continuously extend EVALPERF'; there is no analysis of how quickly benchmark utility will degrade as models improve or as tasks become memorized.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "Failure modes of the benchmark itself (e.g., gaming via memorized solutions, instruction-count as proxy breaking down, Python-only coverage) are not systematically discussed; DPS cost is acknowledged but framed positively.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The full pipeline and evaluator are open-sourced at github.com/evalplus/evalplus, and reference solutions per cluster are embedded in the dataset to enable reproduction of reported DPS scores.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": false,
           "justification": "No formal data card is provided; while the curation methodology is described in detail, there is no structured documentation of dataset statistics, splits, preprocessing edge cases, or known biases.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark is released as part of EvalPlus on GitHub, but no license is mentioned in the paper, leaving reuse terms unclear.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The paper explains what the benchmark is for but does not explicitly state what conclusions should NOT be drawn from EVALPERF results (e.g., no guidance on extrapolating to non-Python languages or non-algorithmic tasks).",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "SAS generates performance-exercising inputs that result in 4.8× more tasks passing all quality filters compared to EvalPlus inputs.",
       "evidence": "Table 1: SAS yields 121 qualifying tasks vs. 25 for EvalPlus after applying the ≥4-cluster criterion on 342 tasks with ≥10 solutions.",
       "supported": "strong"
     },
     {
       "claim": "Scaling laws hold for code correctness but not for code efficiency.",
       "evidence": "Figure 5 shows 7/12 pairs where larger models outperform on DPS, but 4 pairs show >1% degradation; the result is mixed, not a clean reversal.",
       "supported": "moderate"
     },
     {
       "claim": "Instruction tuning consistently improves code efficiency beyond correctness.",
       "evidence": "Figure 4 shows instruct > base for most model families (e.g., DeepSeekCoder-6.7B: +19% DPS), with StarCoder2-15B as the only clear exception.",
       "supported": "strong"
     },
     {
       "claim": "Performance-encouraging prompts (perf-instruct, perf-CoT) do not consistently improve efficiency and often degrade correctness.",
       "evidence": "Figure 4 shows no consistent advantage for perf-instruct/perf-CoT over instruct; Table 3 shows correctness degradation for most models under performance prompts.",
       "supported": "strong"
     },
     {
       "claim": "EVALPERF produces highly consistent results across platforms (max CV 0.4%).",
       "evidence": "Table 2 shows DPS for three models across four hardware configurations with coefficient of variation ≤0.4% in all cases.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4 Turbo achieves the best DPS but not the best DPSnorm, where DeepSeekCoder-6.7B-instruct leads.",
       "evidence": "Table 3 and Figure 8: GPT-4 Turbo avg DPS 88.5–91.5 vs. DeepSeekCoder-6.7B-instruct avg DPSnorm 81.4 being highest across models.",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval"
   ],
   "key_findings": "DPE successfully addresses two core failures of existing coding benchmarks — insufficient computational load and misleading speedup metrics — by synthesizing performance-exercising test inputs and introducing the Differential Performance Score (DPS). Instruction tuning consistently improves both code correctness and efficiency across model families, while scaling laws that hold for correctness do not cleanly extend to code efficiency. EVALPERF achieves near-zero cross-platform variance (CV ≤ 0.4%) by using hardware instruction counters and relative performance ranking, making it reliably reproducible across diverse hardware configurations.",
   "red_flags": [
     {
       "flag": "No contamination analysis",
       "detail": "EVALPERF tasks originate from HumanEval and MBPP, both widely used public benchmarks almost certainly in the training data of evaluated models; no analysis of whether models have memorized these tasks is presented."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks a dedicated limitations or threats-to-validity section; key scope restrictions (Python-only, 121 tasks, LLM-as-oracle for reference solutions) are not framed as limitations."
     },
     {
       "flag": "No human baseline",
       "detail": "A benchmark for code efficiency evaluation never establishes how humans perform, making it impossible to contextualize LLM scores relative to human capability."
     },
     {
       "flag": "Broad scaling law claim from limited evidence",
       "detail": "The claim that 'scaling law fails for code efficiency' is based on 12 within-family pairwise comparisons across three model families; 7/12 pairs actually show larger-is-better, making the headline finding overstated."
     },
     {
       "flag": "Python-only benchmark with language-general framing",
       "detail": "All 121 tasks are Python; the framework is described as general, but no validation in other languages is provided to support that generalization."
     },
     {
       "flag": "No licensing information",
       "detail": "The benchmark is released on GitHub but no license is stated in the paper, leaving downstream use rights ambiguous."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code (HumanEval)",
       "relevance": "Primary source benchmark from which EVALPERF tasks are derived; canonical correctness evaluation baseline."
     },
     {
       "title": "Program Synthesis with Large Language Models (MBPP)",
       "relevance": "Second primary source benchmark for EVALPERF tasks; used for pass@1 correctness evaluation alongside HumanEval+."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation (EvalPlus)",
       "relevance": "Framework providing rigorous correctness tests used as prerequisite filter; also serves as SAS baseline comparison."
     },
     {
       "title": "Learning Performance-Improving Code Edits (PIE)",
       "relevance": "Closest prior work on LLM code efficiency evaluation; DPE differentiates itself from PIE's program optimization setting and simulator-based profiling."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "relevance": "Sibling benchmark addressing contamination in coding evaluation; motivates the contamination problem DPE does not fully address."
     },
     {
       "title": "EffiBench: Benchmarking the Efficiency of Automatically Generated Code",
       "relevance": "Contemporaneous sibling benchmark for code efficiency; DPE claims to address its limitation of variation-sensitive speedup metrics."
     },
     {
       "title": "ECCO: Can We Improve Model-Generated Code Efficiency without Sacrificing Functional Correctness?",
       "relevance": "Another contemporaneous sibling benchmark for code efficiency evaluation mentioned in related work comparison."
     },
     {
       "title": "ALGO: Synthesizing Algorithmic Programs with Generated Oracle Verifiers",
       "relevance": "Prior work using LLM-generated test input generators; DPE's SAS approach is contrasted against ALGO's reliance on ChatGPT Code Interpreter."
     },
     {
       "title": "Chain-of-Thought Prompting Elicits Reasoning in Large Language Models",
       "relevance": "Foundational technique used by SAS for few-shot CoT input generator synthesis."
     },
     {
       "title": "Scaling Laws for Neural Language Models",
       "relevance": "Established scaling law that DPE's experiments find does not hold for code efficiency, a central empirical finding."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 3,
       "justification": "EVALPERF is open-sourced, actively maintained as part of EvalPlus, and directly usable by practitioners to benchmark LLM code efficiency."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that scaling laws fail for code efficiency while holding for correctness challenges common assumptions about larger models being uniformly better."
     },
     "fear_safety": {
       "score": 0,
       "justification": "No AI safety or risk concerns; the paper is focused on performance optimization evaluation methodology."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Mild challenge to existing benchmark methodology (HumanEval, MBPP inadequacy for efficiency), but framed constructively rather than controversially."
     },
     "demo_ability": {
       "score": 3,
       "justification": "Fully open-sourced benchmark and pipeline at github.com/evalplus/evalplus; anyone can run EVALPERF on their model immediately."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "EvalPlus community has some recognition in the code generation space, but no major lab affiliation; published at COLM 2024, a newer venue."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "41567138",
         "title": "Can Generative Multi-Agents Spontaneously Form a Society?",
         "points": 48,
         "comments": 5,
         "url": "https://news.ycombinator.com/item?id=41567138",
         "created_at": "2024-09-17T12:55:14Z"
       },
       {
         "hn_id": "43921813",
         "title": "Human-Like Episodic Memory for Infinite Context LLMs",
         "points": 27,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43921813",
         "created_at": "2025-05-08T00:21:21Z"
       },
       {
         "hn_id": "40021906",
         "title": "Wu's Method Can Boost AlphaGeometry to Outperform Gold Medalists at IMO Geometry",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40021906",
         "created_at": "2024-04-13T10:06:11Z"
       },
       {
         "hn_id": "24247130",
         "title": "Manticore: A 4096-core RISC-V Chiplet Arch for Ultra-efficient FP Computing",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=24247130",
         "created_at": "2020-08-22T20:45:30Z"
       },
       {
         "hn_id": "40015493",
         "title": "Show HN: Symbolic AI at Silver Medal, Boosts AlphaGeometry to Beat IMO Geo Gold",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40015493",
         "created_at": "2024-04-12T17:36:41Z"
       },
       {
         "hn_id": "39691144",
         "title": "Adding NVMe SSDs to Enable and Accelerate 100B Model Fine-Tuning on a Single GPU",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39691144",
         "created_at": "2024-03-13T13:44:20Z"
       },
       {
         "hn_id": "41317807",
         "title": "Human-Like Episodic Memory for Infinite Context LLMs",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41317807",
         "created_at": "2024-08-22T07:52:12Z"
       },
       {
         "hn_id": "40001562",
         "title": "Symbolic AI at Silver Medal, Boosts AlphaGeometry to Beat Gold at IMO Geometry",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40001562",
         "created_at": "2024-04-11T12:53:16Z"
       },
       {
         "hn_id": "37165307",
         "title": "Taboo and Collaborative Knowledge Production: Evidence from Wikipedia",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37165307",
         "created_at": "2023-08-17T17:34:16Z"
       },
       {
         "hn_id": "24208779",
         "title": "Manticore: A 4096-core RISC-V Chiplet Arch for Ultra-efficient FP Computing",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=24208779",
         "created_at": "2020-08-19T10:00:12Z"
       }
     ],
     "top_points": 48,
     "total_points": 109,
     "total_comments": 9
   }
 }