ai-research-survey

scan.json (28046B)

---

 {
   "paper": {
     "title": "Evaluating Language Models for Efficient Code Generation",
     "authors": [
       "Jiawei Liu",
       "Songrun Xie",
       "Junhao Wang",
       "Yuxiang Wei",
       "Yifeng Ding",
       "Lingming Zhang"
     ],
     "year": 2024,
     "venue": "COLM 2024",
     "arxiv_id": "2408.06450",
     "doi": "10.48550/arXiv.2408.06450"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "The paper introduces DPE (Differential Performance Evaluation) and creates EVALPERF, a benchmark of 121 performance-exercising coding tasks for evaluating LLM code efficiency. Key findings include: instruction tuning improves both code correctness and efficiency (e.g., 19% DPS improvement for DeepSeekCoder-6.7B), the scaling law does not reliably hold for code efficiency (performance degradation in 4/12 size comparisons), performance-encouraging prompts do not consistently improve efficiency, and EVALPERF achieves highly consistent cross-platform evaluation (CV ≤ 0.4%) via hardware performance counters.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper states in §1 contribution 3: 'we also fully open-source and maintain the data curation pipeline and evaluator at github.com/evalplus/evalplus as part of EvalPlus.'"
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "EVALPERF with 121 tasks, test inputs, and reference solutions is released as part of the EvalPlus open-source project at github.com/evalplus/evalplus."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "While hardware configurations are listed in Table 2 (CPUs, RAM), no software environment specifications (requirements.txt, Python version, library versions, Dockerfile) are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The open-source repository is mentioned but no specific commands or reproduction guide is included in the paper text."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Table 3 report Avg/Max/Min across samples, and Table 2 reports CV across platforms, but no confidence intervals or error bars are provided for main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes numerous comparative claims (e.g., 'instruction-tuned models also tends to be more efficient') based solely on comparing DPS numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports magnitude of effects with context, e.g., 'instruction-tuned DeepSeekCoder-6.7B improves the base model by 19% regarding DPS' (§4.1), 'relative improvement of 4.8×' for SAS vs EvalPlus (§4.2)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for the choice of 121 tasks, 50 samples per model, or 10 samples for GPT-4. The 10-sample limit for GPT-4 is attributed to 'cost mitigation' without justifying adequacy."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 reports Avg, Max, and Min DPS/DPSnorm across the first 10 correct samples per task, providing a range measure. Table 2 reports Coefficient of Variation across platforms."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Multiple model families are compared (CodeLlama, DeepSeekCoder, StarCoder, StarCoder2, GPT-4 Turbo). SAS is compared against EvalPlus as a baseline in Table 1."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4 Turbo, StarCoder2, DeepSeekCoder, CodeQwen1.5, and CodeLlama, all contemporary models at the time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation systematically studies the impact of individual factors: instruction tuning (base vs instruct), prompting strategies (instruct vs perf-instruct vs perf-CoT), and model sizes (within families), each controlling other variables."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Three metrics are used: DPS (Differential Performance Score), DPSnorm (normalized DPS), and pass@1 for correctness. Results are reported for all three in Table 3."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated via test execution and hardware performance counters. No human evaluation of code quality or efficiency is included."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "EVALPERF tasks and generated test inputs are independent of the models being evaluated. Pairwise efficiency comparison is done over the common set of passing solutions to control for correctness differences."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model, per variant (base/instruct/perf-instruct/perf-CoT), per model size in Figures 4-5 and Table 3. Pairwise comparison matrices are provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where the scaling law fails for efficiency (§4.1, Figure 5, e.g., 6% degradation from StarCoder2 3B to 7B), and where performance-encouraging prompts degrade correctness (Table 3)."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: 'performance-encouraging prompts neither consistently nor noticeably improve the code efficiency' (§4.1), scaling law doesn't hold for efficiency (4 of 12 pairs show degradation), and perf-prompts 'commonly lead to correctness degradation' (§4.1/Table 3)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about DPE framework, instruction tuning benefiting efficiency, scaling law not holding for efficiency, and 4.8× improvement over prior art inputs are all supported by results in §4.1 and §4.2."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'instruction tuning benefits efficiency' and 'model size does not reliably improve efficiency' are supported by controlled comparisons within model families where only the variable of interest differs (e.g., same model, base vs instruct; same family, different sizes)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Evaluating Language Models for Efficient Code Generation' broadly, but experiments are limited to Python tasks from HumanEval+ and MBPP+. The abstract and conclusion make general claims about 'code generation' without bounding to Python or to algorithmic-style programming problems."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the key findings. For example, the instruction tuning efficiency benefit could stem from training data quality rather than the tuning process itself, and the scaling law failure could have multiple explanations, none of which are explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly distinguishes between their proxy metric (hardware instruction count / DPS) and actual runtime efficiency. §2 and Appendix A.2 discuss limitations of runtime measurement and why DPS provides a more meaningful compound metric than raw speedup, carefully defining what DPS measures."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are given: 'gpt-4-0125-preview' for GPT-4 Turbo, 'AWQ-quantized DeepSeekCoder-instruct-33B' for input generation, and specific model family sizes (CodeLlama-7B/13B/34B/70B, DeepSeekCoder-1.3B/6.7B/33B, etc.)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "The SAS prompt is shown in Figure 2 with actual text and few-shot examples. Evaluation prompts are described with actual text: perf-instruct uses 'solve the programming task efficiently by writing a fast implementation', perf-CoT adds 'Think step by step'. Base prompts come from public HumanEval/MBPP."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature (0.8 for generation, 0.2 for evaluation), number of samples (50, 10 for GPT-4), scale factor (starting at 2^1, exponential), time wall (20s), memory wall (16GB), clustering parameters (bias=20%, w=10k, K=4), and tthresh=10k instructions are all reported in §3."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation involves direct code generation from LLMs with prompts, with no multi-step agents, tools, or feedback loops."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented in detail: solution curation from 21 LLMs (§2.1), input generator synthesis via SAS (§2.2), filtering criteria with counts (Table 1: 563 → 342 → 271 → 121), and clustering methodology (§2.4, Figure 3)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The paper has sections for Introduction, DPE, EVALPERF, Evaluation, Related Work, Conclusion, and Appendix, with no explicit limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No specific threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "No explicit scope boundaries are stated. The paper does not discuss what the results do NOT show, what settings were excluded, or what claims the authors are not making."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (tasks, test inputs, reference solutions, performance clusters) is released as open source at github.com/evalplus/evalplus, enabling independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Solution collection is described in §2.1 and §3: 'we sample and test code solutions from 21 open LLMs that achieve over a pass@1 score of 50 on the EvalPlus leaderboard, where we sample 50 solutions for each model at a temperature of 0.8.'"
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are LLM-generated code on standard public benchmarks (HumanEval+ and MBPP+)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented across §§2.1-2.5 and §3: valid solution curation → SAS input generation → exponential input sampling → task selection with filtering criteria → adaptive clustering. Table 1 shows counts at each filtering stage."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgment section states: 'This work was partially supported by NSF grant CCF-2131943 and Kwai Inc, as well as API credits from the OpenAI Researcher Access Program.'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Illinois Urbana-Champaign and Tongji University. No undisclosed industry affiliations are apparent."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "OpenAI provided API credits through the Researcher Access Program, and the paper evaluates GPT-4 Turbo (an OpenAI product), which achieves the best DPS in the evaluation. OpenAI has a financial interest in GPT-4 performing well."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial disclosure statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models. The models are used without specifying when their training data was collected."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) are public benchmarks that likely appear in training data of models released in 2023-2024. No analysis of potential overlap is provided."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP were published in 2021 and are widely known public benchmarks. All evaluated models were trained after 2021 and may have seen these tasks. The paper cites Jain et al. (2024) on contamination importance but does not address it for their own evaluation."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The paper evaluates LLM code generation via automated benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All evaluation is automated using LLM-generated code and hardware profiling."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions evaluation takes 'approximately no more than 15 minutes' for profiling, but does not report the API cost or compute cost of generating 50 solutions per model across 21 LLMs, nor the cost of running the SAS pipeline."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper does not report GPU hours, total API spend, or total compute used for solution sampling from 21 models, input generation, or profiling."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. Solutions are sampled at fixed temperatures (0.2 or 0.8) but results are not reported across different random seeds."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states: 50 samples at temperature 0.2 (10 for GPT-4), first 10 correct solutions evaluated for efficiency. Appendix A.1 states '5 times' for runtime profiling. 16 input generator samples per task."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Clustering hyperparameters (bias=20%, w=10k, K=4, tthresh=10k) are stated but no justification or search budget for how these values were selected. No alternatives were reported or discussed."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The threshold values for task filtering and clustering (bias=20%, w=10k) are stated without justification for why these specific values were chosen over alternatives."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical significance tests are performed, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors create and evaluate on their own benchmark (EVALPERF) without acknowledging potential bias in evaluating their own system. No independent evaluation or discussion of author-evaluation bias."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models ranging from 1.3B to 70B parameters are compared on efficiency without normalizing for compute budget. The paper compares model sizes (Figure 5) but does not report performance as a function of inference compute cost."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses measurement validity (why instruction count is better than runtime, why DPS is better than speedup) but does not examine whether algorithmic-style HumanEval/MBPP tasks reflect real-world code efficiency challenges."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. All evaluations use direct code generation from LLMs without agentic scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published years before the evaluated models were trained. Models trained after 2021 may have seen solutions to these tasks. This temporal leakage is not discussed."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks information. The task descriptions from HumanEval/MBPP include function signatures and docstrings that may appear in training data."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether train and test examples share structural similarities. HumanEval/MBPP tasks are well-known and solutions circulate widely online."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No leakage detection or prevention method is applied. No canary strings, membership inference tests, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "General instruction tuning benefits both code correctness and efficiency, even though existing instruction tuning methods were not designed to optimize code efficiency.",
       "evidence": "Figure 4 shows pairwise DPS comparisons where instruct variants generally outperform base variants. DeepSeekCoder-6.7B-instruct improves base by 19% DPS. Exception: StarCoder2-15B where base slightly outperforms instruct.",
       "supported": "moderate"
     },
     {
       "claim": "The scaling law persists for code correctness but does not seem explicit for code efficiency.",
       "evidence": "Figure 5 shows mixed results across model sizes: 7/12 pairs favor larger models, but 4 pairs show >1% degradation. StarCoder2 shows 6% degradation from 3B to 7B. CodeLlama shows inconsistent scaling.",
       "supported": "moderate"
     },
     {
       "claim": "Performance-encouraging prompts neither consistently nor noticeably improve code efficiency.",
       "evidence": "Figure 4 shows small/inconsistent DPS differences between instruct, perf-instruct, and perf-CoT variants. Table 3 shows perf-prompts commonly lead to correctness degradation.",
       "supported": "strong"
     },
     {
       "claim": "SAS creates inputs that are more performance-exercising than EvalPlus by 4.8×.",
       "evidence": "Table 1: after all filtering, SAS retains 121 tasks vs EvalPlus's 25 tasks. SAS passes 271/342 tasks (79%) on computation criterion vs EvalPlus's 204 (60%).",
       "supported": "strong"
     },
     {
       "claim": "EVALPERF leads to consistent performance evaluation across various platforms with maximum CV of 0.4%.",
       "evidence": "Table 2 shows DPS and DPSnorm for 3 models across 4 hardware platforms (desktop, workstation, server) with CVs ranging from 0.1% to 0.4%.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No contamination discussion",
       "detail": "HumanEval (2021) and MBPP (2021) are well-known public benchmarks that are likely in the training data of all evaluated models (released 2023-2024). The paper cites contamination-related work (Jain et al. 2024) but does not address contamination risk in its own evaluation. This is especially concerning because better-contaminated models would appear both more correct and potentially more efficient."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper has no limitations, threats-to-validity, or scope-bounding discussion. Key unacknowledged limitations include: Python-only evaluation with broad claims about 'code generation', competition-style tasks vs real-world efficiency challenges, and potential selection bias from using models that score >50 pass@1 on EvalPlus."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims ('instruction tuning benefits efficiency', 'scaling law fails for efficiency') are based on comparing raw DPS numbers without any statistical tests, despite comparing across noisy samples with temperature-based generation."
     },
     {
       "flag": "OpenAI funding with GPT-4 evaluation",
       "detail": "OpenAI provided API credits via the Researcher Access Program, and GPT-4 Turbo achieves the best DPS score in the evaluation. While the paper's main contribution is the benchmark framework rather than a model ranking, the best-performing model is from a funder."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "relevance": "Introduces HumanEval, the foundational code generation benchmark used as a source for EVALPERF."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell I. Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces MBPP benchmark for code generation, used as a source for EVALPERF."
     },
     {
       "title": "Is your code generated by chatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "Introduces EvalPlus with extended test suites (HumanEval+ and MBPP+), the direct foundation for EVALPERF's task pool."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["Naman Jain", "King Han", "Alex Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses contamination in code evaluation, directly relevant to benchmark validity concerns."
     },
     {
       "title": "Learning performance-improving code edits",
       "authors": ["Alexander Shypula", "Aman Madaan", "Yimeng Zeng"],
       "year": 2023,
       "relevance": "PIE benchmark for code optimization with CPU simulators, the closest prior work to EVALPERF for efficiency evaluation."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "relevance": "Major open-source code LLM family evaluated in this paper across 4 sizes."
     },
     {
       "title": "DeepSeek Coder: Let the code write itself",
       "authors": ["DeepSeek AI"],
       "year": 2023,
       "relevance": "Open-source code LLM family evaluated in this paper; also used for SAS input generation."
     },
     {
       "title": "StarCoder 2 and The Stack v2: The next generation",
       "authors": ["Anton Lozhkov", "Raymond Li", "Loubna Ben Allal"],
       "year": 2024,
       "arxiv_id": "2402.19173",
       "relevance": "Open-source code LLM family evaluated in this paper; evaluation setup follows their protocol."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan"],
       "year": 2022,
       "relevance": "Evaluates security aspects of LLM-generated code, relevant to comprehensive code quality evaluation."
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "authors": ["Dong Huang", "Jie M Zhang", "Yuhao Qing", "Heming Cui"],
       "year": 2024,
       "arxiv_id": "2402.02037",
       "relevance": "Concurrent benchmark for code efficiency evaluation, compared with DPE in the related work section."
     },
     {
       "title": "Magicoder: Empowering code generation with OSS-instruct",
       "authors": ["Yuxiang Wei", "Zhe Wang", "Jiawei Liu", "Yifeng Ding", "Lingming Zhang"],
       "year": 2024,
       "relevance": "Code instruction tuning method from overlapping authors, relevant to findings about instruction tuning improving efficiency."
     }
   ]
 }