ai-research-survey

scan-v4.json (21918B)

---

 {
   "scan_version": 4,
   "paper_type": "benchmark-creation",
   "paper": {
     "title": "Evaluating Language Models for Efficient Code Generation",
     "authors": [
       "Jiawei Liu",
       "Songrun Xie",
       "Junhao Wang",
       "Yuxiang Wei",
       "Yifeng Ding",
       "Lingming Zhang"
     ],
     "year": 2024,
     "venue": "COLM 2024",
     "arxiv_id": "2408.06450",
     "doi": "10.48550/arXiv.2408.06450"
   },
   "checklist": {
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about DPE framework, instruction tuning benefiting efficiency, scaling law not holding for efficiency, and 4.8× improvement over prior art inputs are all supported by results in §4.1 and §4.2.",
         "source": "opus"
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'instruction tuning benefits efficiency' and 'model size does not reliably improve efficiency' are supported by controlled comparisons within model families where only the variable of interest differs (e.g., same model, base vs instruct; same family, different sizes).",
         "source": "opus"
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Evaluating Language Models for Efficient Code Generation' broadly, but experiments are limited to Python tasks from HumanEval+ and MBPP+. The abstract and conclusion make general claims about 'code generation' without bounding to Python or to algorithmic-style programming problems.",
         "source": "opus"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the key findings. For example, the instruction tuning efficiency benefit could stem from training data quality rather than the tuning process itself, and the scaling law failure could have multiple explanations, none of which are explored.",
         "source": "opus"
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between their proxy metric (hardware instruction count / DPS) and actual runtime efficiency. §2 and Appendix A.2 discuss limitations of runtime measurement and why DPS provides a more meaningful compound metric than raw speedup, carefully defining what DPS measures.",
         "source": "opus"
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The paper has sections for Introduction, DPE, EVALPERF, Evaluation, Related Work, Conclusion, and Appendix, with no explicit limitations discussion.",
         "source": "opus"
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper.",
         "source": "opus"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were excluded, or what claims the authors are not making.",
         "source": "opus"
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section states: 'This work was partially supported by NSF grant CCF-2131943 and Kwai Inc, as well as API credits from the OpenAI Researcher Access Program.'",
         "source": "opus"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Illinois Urbana-Champaign and Tongji University. No undisclosed industry affiliations are apparent.",
         "source": "opus"
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provided API credits through the Researcher Access Program, and the paper evaluates GPT-4 Turbo (an OpenAI product), which achieves the best DPS in the evaluation. OpenAI has a financial interest in GPT-4 performing well.",
         "source": "opus"
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper.",
         "source": "opus"
       }
     },
     "scope_and_framing": {
       "key_terms_defined": {
         "applies": true,
         "answer": true,
         "justification": "Key terms are defined precisely: 'code efficiency' is scoped to execution efficiency (not readability), DPS is formally defined with its cumulative ratio formula (§2.5), and 'performance-exercising' inputs are defined via the filtering criteria (§2.3).",
         "source": "haiku"
       },
       "intended_contribution_clear": {
         "applies": true,
         "answer": true,
         "justification": "Four contributions are explicitly enumerated in §1: a new evaluation dimension (efficiency), the DPE technique, the EVALPERF benchmark, and an empirical study—each clearly scoped.",
         "source": "haiku"
       },
       "engagement_with_prior_work": {
         "applies": true,
         "answer": true,
         "justification": "§5 engages substantively with HumanEval, MBPP, EvalPlus, PIE, and contemporaneous sibling benchmarks (EffiBench, ECCO), explaining specific technical limitations of each and how DPE addresses them rather than just listing citations.",
         "source": "haiku"
       }
     }
   },
   "type_checklist": {
     "benchmark-creation": {
       "construct_design": {
         "construct_validity_argued": {
           "applies": true,
           "answer": true,
           "justification": "The paper argues why DPE measures code efficiency better than alternatives: §2 explains that efficiency-demanding inputs are required because light computation makes all complexities 'equal,' and §A.2 argues that DPS is a more interpretable compound metric than average speedup, which is skewed by outlier tasks.",
           "source": "haiku"
         },
         "difficulty_distribution_characterized": {
           "applies": true,
           "answer": true,
           "justification": "Difficulty is operationalized through performance clustering (§2.4) requiring K≥4 clusters per task, and Table 1 documents the filtering funnel. The adaptive clustering algorithm produces tasks with distinct performance tiers rather than assuming difficulty.",
           "source": "haiku"
         },
         "ceiling_floor_effects_checked": {
           "applies": true,
           "answer": false,
           "justification": "The benchmark is designed against ceiling/floor effects through filtering (sufficient computation >10k instructions, ≥4 performance clusters), but the paper does not explicitly report or analyze ceiling/floor effects at the model-performance level after benchmark construction.",
           "source": "haiku"
         },
         "human_baseline_included": {
           "applies": true,
           "answer": false,
           "justification": "No human baseline is included or discussed. The evaluation covers only LLM variants; there is no measurement of how human programmers perform on EVALPERF tasks.",
           "source": "haiku"
         },
         "scoring_rubric_justified": {
           "applies": true,
           "answer": true,
           "justification": "DPS is formally defined (§2.5) and extensively justified against average speedup in §A.2, including a concrete counterexample showing how speedup averaging can mislead. Edge cases (DPS=0 for failing or slowest solutions) are addressed in the definition.",
           "source": "haiku"
         }
       },
       "robustness": {
         "contamination_resistance_designed": {
           "applies": true,
           "answer": false,
           "justification": "EVALPERF tasks are derived from HumanEval+ and MBPP+, both widely used training data sources. The paper cites LiveCodeBench's contamination-free design in related work but applies no contamination resistance (temporal splits, canary strings, or dynamic generation) to EVALPERF itself.",
           "source": "haiku"
         },
         "temporal_robustness_discussed": {
           "applies": true,
           "answer": false,
           "justification": "The paper mentions future plans to extend EVALPERF (§3) but does not discuss whether or when models will saturate the benchmark, what update cadence is planned, or what mechanisms would prevent gaming as models improve.",
           "source": "haiku"
         },
         "failure_modes_discussed": {
           "applies": true,
           "answer": false,
           "justification": "§A.2 discusses measurement trade-offs (hardware counters vs. simulators vs. physical runtime) and related work critiques sibling benchmarks, but the paper does not systematically discuss what EVALPERF itself fails to measure or how it could be gamed.",
           "source": "haiku"
         },
         "baseline_implementations_provided": {
           "applies": true,
           "answer": true,
           "justification": "The paper explicitly states: 'we fully open-source and maintain the data curation pipeline and evaluator at github.com/evalplus/evalplus,' providing reference solutions, profiling code, and the evaluation framework.",
           "source": "haiku"
         }
       },
       "documentation": {
         "dataset_documentation_complete": {
           "applies": true,
           "answer": true,
           "justification": "The data curation pipeline is documented in detail: source tasks (HumanEval+ + MBPP+), solution sampling (21 LLMs, 50 samples each), input generator creation (DeepSeekCoder-33B, 16 samples), filtering criteria with explicit thresholds, and clustering parameters are all specified.",
           "source": "haiku"
         },
         "licensing_and_access_clear": {
           "applies": true,
           "answer": false,
           "justification": "While the benchmark is stated to be open-sourced at github.com/evalplus/evalplus, the paper itself does not specify a license for EVALPERF or clarify under what terms others may use, modify, or redistribute it.",
           "source": "haiku"
         },
         "intended_use_specified": {
           "applies": true,
           "answer": false,
           "justification": "The intended use (evaluating LLM code efficiency on algorithmic Python tasks) is clear from context, but the paper does not state what should NOT be concluded from EVALPERF results—for example, that DPS scores don't reflect real-world performance on non-algorithmic tasks or other languages.",
           "source": "haiku"
         }
       }
     }
   },
   "claims": [
     {
       "claim": "DPE generates inputs that are 4.8× more performance-exercising than EvalPlus baselines, measured by tasks passing the clustering criterion",
       "evidence": "Table 1: SAS passes 121 tasks with ≥4 clusters vs. EvalPlus's 25 tasks (271 vs. 204 with computation filter alone)",
       "supported": "strong"
     },
     {
       "claim": "EVALPERF produces consistent evaluation results across diverse hardware platforms, with maximum coefficient of variation <0.4%",
       "evidence": "Table 2 shows DPS scores for three models across four test beds (desktop i7, desktop i9-12900K, workstation, server) with CV ranging 0.1–0.4%",
       "supported": "strong"
     },
     {
       "claim": "General instruction tuning improves code efficiency alongside correctness, even though it was not designed to optimize efficiency",
       "evidence": "Figure 4 shows instruct variants outperform base models for most model families (e.g., DeepSeekCoder-6.7B: +19% DPS), with StarCoder2-15B as the only exception",
       "supported": "moderate"
     },
     {
       "claim": "The scaling law does not hold for code efficiency — larger models within a family do not consistently generate more efficient code",
       "evidence": "Figure 5 shows 7/12 pairs where larger models outperform smaller, but 4 cases of >1% degradation including StarCoder2 3B→7B (−6%)",
       "supported": "moderate"
     },
     {
       "claim": "Performance-encouraging prompts (perf-instruct, perf-CoT) do not consistently or noticeably improve code efficiency",
       "evidence": "Figure 4 shows no systematic advantage for perf-instruct or perf-CoT over instruct prompting; Table 3 shows performance-encouraging prompts commonly cause correctness degradation",
       "supported": "moderate"
     },
     {
       "claim": "GPT-4 Turbo achieves the highest DPS among evaluated models",
       "evidence": "Table 3 shows GPT-4 Turbo reaching 91.5% avg DPS (perf-CoT), highest of all models; Figure 7 visualizes this",
       "supported": "strong"
     }
   ],
   "methodology_tags": [
     "benchmark-eval",
     "benchmark-creation"
   ],
   "key_findings": "DPE (Differential Performance Evaluation) addresses two core failures of existing code benchmarks—light-computation tasks and misleading speedup metrics—by using LLM-synthesized performance-exercising inputs and a percentile-rank DPS metric. EVALPERF (121 Python tasks) is 4.8× more discriminating than EvalPlus baselines and produces consistent rankings across hardware platforms (CV <0.4%). Instruction tuning reliably improves code efficiency alongside correctness, but the scaling law that holds for correctness does not extend to efficiency, and performance-encouraging prompts provide no reliable efficiency gains.",
   "red_flags": [
     {
       "flag": "Contamination not addressed",
       "detail": "EVALPERF tasks are drawn from HumanEval+ and MBPP+, both widely used LLM training sources. The paper cites contamination-free benchmarks in related work but applies no contamination mitigation to EVALPERF itself."
     },
     {
       "flag": "Causal language for observational findings",
       "detail": "'General instruction tuning benefits both code correctness and efficiency' is presented causally, but base and instruct variants differ in training data composition and volume, not just instruction tuning—confounds are uncontrolled."
     },
     {
       "flag": "Python-only generalization",
       "detail": "All 121 tasks are Python algorithmic problems from HumanEval/MBPP lineage. Findings about scaling laws and instruction tuning are presented as general conclusions without bounding to Python or algorithmic tasks."
     },
     {
       "flag": "No human baseline",
       "detail": "There is no measurement of how human programmers perform on EVALPERF tasks, making it impossible to calibrate where LLM scores fall relative to human-level efficiency."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper contains no dedicated limitations or threats-to-validity discussion. Implicit limitations (Python only, HumanEval/MBPP task distribution, hardware counter limitations for interpreted languages) are not systematically acknowledged."
     }
   ],
   "cited_papers": [
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation (EvalPlus)",
       "relevance": "Direct predecessor; EVALPERF uses EvalPlus's rigorous tests for correctness validation and EvalPlus inputs as a baseline comparison for performance difficulty"
     },
     {
       "title": "Evaluating large language models trained on code (HumanEval)",
       "relevance": "Source of 164 base tasks used to construct EVALPERF; the standard code generation correctness benchmark"
     },
     {
       "title": "Program synthesis with large language models (MBPP)",
       "relevance": "Source of 399 base tasks used to construct EVALPERF alongside HumanEval+"
     },
     {
       "title": "Learning performance-improving code edits (PIE)",
       "relevance": "Most closely related benchmark for code efficiency evaluation; DPE explicitly compares its methodology against PIE's C++ optimization focus and simulator-based profiling"
     },
     {
       "title": "LiveCodeBench: Holistic and contamination-free evaluation of LLMs for code",
       "relevance": "Contamination-aware benchmark cited as addressing a limitation that EVALPERF does not address"
     },
     {
       "title": "Chain-of-thought prompting elicits reasoning in large language models",
       "relevance": "Foundational technique used by DPE's Synthesizing a Synthesizer (SAS) for input generator creation"
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "relevance": "Contemporaneous sibling benchmark for code efficiency evaluation; DPE claims to address its limitations"
     },
     {
       "title": "How efficient is LLM-generated code? A rigorous & high-standard benchmark",
       "relevance": "Another contemporaneous efficiency benchmark; DPE contrasts its metric design against speedup-based approaches used here"
     },
     {
       "title": "ALGO: Synthesizing algorithmic programs with generated oracle verifiers",
       "relevance": "Related work on LLM-generated test input synthesis; SAS is explicitly differentiated from ALGO's approach"
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Practitioners evaluating LLMs for code generation can directly use the open-sourced EVALPERF benchmark and DPE framework to assess efficiency, a gap not addressed by existing benchmarks."
     },
     "surprise_contrarian": {
       "score": 2,
       "justification": "The finding that scaling laws don't hold for code efficiency—while they hold for correctness—challenges a dominant assumption in LLM scaling research."
     },
     "fear_safety": {
       "score": 0,
       "justification": "The paper raises no AI safety or risk concerns; it is a methodological contribution to benchmark design."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "The paper critiques existing benchmarks (EvalPlus, contemporaneous efficiency benchmarks) as inadequate, but the critique is technical rather than controversial."
     },
     "demo_ability": {
       "score": 3,
       "justification": "EVALPERF and the full DPE pipeline are fully open-sourced at github.com/evalplus/evalplus; practitioners can run evaluations immediately."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "UIUC is a well-known institution and EvalPlus has prior community recognition, but no major industry lab (Google, OpenAI, Meta) is the primary author."
     }
   },
   "hn_data": {
     "threads": [
       {
         "hn_id": "41567138",
         "title": "Can Generative Multi-Agents Spontaneously Form a Society?",
         "points": 48,
         "comments": 5,
         "url": "https://news.ycombinator.com/item?id=41567138",
         "created_at": "2024-09-17T12:55:14Z"
       },
       {
         "hn_id": "43921813",
         "title": "Human-Like Episodic Memory for Infinite Context LLMs",
         "points": 27,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=43921813",
         "created_at": "2025-05-08T00:21:21Z"
       },
       {
         "hn_id": "40021906",
         "title": "Wu's Method Can Boost AlphaGeometry to Outperform Gold Medalists at IMO Geometry",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=40021906",
         "created_at": "2024-04-13T10:06:11Z"
       },
       {
         "hn_id": "24247130",
         "title": "Manticore: A 4096-core RISC-V Chiplet Arch for Ultra-efficient FP Computing",
         "points": 7,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=24247130",
         "created_at": "2020-08-22T20:45:30Z"
       },
       {
         "hn_id": "40015493",
         "title": "Show HN: Symbolic AI at Silver Medal, Boosts AlphaGeometry to Beat IMO Geo Gold",
         "points": 6,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40015493",
         "created_at": "2024-04-12T17:36:41Z"
       },
       {
         "hn_id": "39691144",
         "title": "Adding NVMe SSDs to Enable and Accelerate 100B Model Fine-Tuning on a Single GPU",
         "points": 3,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=39691144",
         "created_at": "2024-03-13T13:44:20Z"
       },
       {
         "hn_id": "41317807",
         "title": "Human-Like Episodic Memory for Infinite Context LLMs",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=41317807",
         "created_at": "2024-08-22T07:52:12Z"
       },
       {
         "hn_id": "40001562",
         "title": "Symbolic AI at Silver Medal, Boosts AlphaGeometry to Beat Gold at IMO Geometry",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=40001562",
         "created_at": "2024-04-11T12:53:16Z"
       },
       {
         "hn_id": "37165307",
         "title": "Taboo and Collaborative Knowledge Production: Evidence from Wikipedia",
         "points": 3,
         "comments": 0,
         "url": "https://news.ycombinator.com/item?id=37165307",
         "created_at": "2023-08-17T17:34:16Z"
       },
       {
         "hn_id": "24208779",
         "title": "Manticore: A 4096-core RISC-V Chiplet Arch for Ultra-efficient FP Computing",
         "points": 2,
         "comments": 1,
         "url": "https://news.ycombinator.com/item?id=24208779",
         "created_at": "2020-08-19T10:00:12Z"
       }
     ],
     "top_points": 48,
     "total_points": 109,
     "total_comments": 9
   }
 }