ai-research-survey

scan.json (30438B)

---

 {
   "paper": {
     "title": "EFFI-LEARNER: Enhancing Efficiency of Generated Code via Self-Optimization",
     "authors": [
       "Dong Huang",
       "Jianbo Dai",
       "Han Weng",
       "Puzhen Wu",
       "Yuhao Qing",
       "Heming Cui",
       "Zhijiang Guo",
       "Jie M. Zhang"
     ],
     "year": 2024,
     "venue": "Neural Information Processing Systems",
     "arxiv_id": "2405.15189"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "EFFI-LEARNER improves LLM-generated code efficiency by feeding execution time and memory usage profiles back to the LLM for iterative self-optimization. Across 22 LLMs on EffiBench, HumanEval, and MBPP, the approach achieves substantial efficiency gains (e.g., 87.1% execution time reduction for StarCoder2-15B on EffiBench) with only 0-0.5% pass@1 degradation. The overhead profile feedback is critical — without it, naive self-refinement often makes code worse (up to 523% TMU increase for CodeLlama-70B). Most efficiency gains occur in the first 1-2 optimization steps, with diminishing returns thereafter.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'The source code of EFFI-LEARNER was released in https://github.com/huangd1999/EffiLearner' and the NeurIPS checklist confirms code was uploaded in supplementary files."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses publicly available benchmarks: EffiBench, HumanEval, MBPP, and EvalPlus (HumanEval-Plus, MBPP-Plus). These are all standard public datasets."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 specifies hardware (Intel Xeon Platinum 8336C CPU, 8× NVIDIA A100-SXM GPUs, 2.0TiB memory) but does not provide software dependency specifications (requirements.txt, library versions, Python version) in the paper itself."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper provides a GitHub link and describes the framework pipeline (Section 3), but does not include step-by-step reproduction instructions with specific commands to replicate the main experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All tables (Tables 1-9) report only point estimates with percentage improvements. No confidence intervals or error bars are provided for any metric."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper repeatedly claims EFFI-LEARNER 'significantly' improves efficiency (e.g., Section 4.3) but no statistical significance tests are used. Comparisons are based solely on raw number differences."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Tables report absolute values before and after optimization with percentage reductions (e.g., 'ET decreases from 0.93 (s) to 0.12 (s) which reduces 87.1% execution time'). Normalized metrics (NET, NMU, NTMU) provide relative comparison to canonical solutions."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The number of evaluation problems is determined by the benchmark sizes (EffiBench, HumanEval, MBPP) without any justification for whether these sample sizes are sufficient for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported. The NeurIPS checklist (Q7) claims greedy decoding makes results 'consistent,' but execution time and memory measurements are inherently variable across runs and no measurement variance is reported."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 compares against Unsupervised Self-Refine, Result-Aware Self-Refine, Memory Profiler only, and Execution Time Profiler only. Table 7 compares with Self-Edit, Critic, PIE, and Supersonic."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include Self-Refine (NeurIPS 2023), Reflexion (NeurIPS 2023), PIE (ICLR 2024), Supersonic, Self-Edit (ACL 2023), and CRITIC (2023). These are recent and relevant approaches."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 ablates the contribution of each profiler component (memory profiler only, execution time profiler only, combined). Table 2 ablates the number of self-optimization steps (0-5)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six metrics are used: Execution Time (ET), Normalized ET (NET), Max Memory Usage (MU), Normalized MU (NMU), Total Memory Usage (TMU), and Normalized TMU (NTMU), defined in Appendix A.5."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "All evaluation is fully automated through execution time profiling and memory profiling. No human evaluation of code quality, readability, or optimization strategies is performed."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1: 'we utilize the open test cases to calculate the efficiency metrics during the self-optimization process, while private test cases provided by EffiBench were used for the final result evaluation.' Similarly, EvalPlus private tests are used for HumanEval/MBPP."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (22 models in Tables 1, 5, 8, 9), per benchmark (EffiBench, HumanEval, MBPP), and per optimization step (Table 2). Individual case studies are also provided."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.6 'Error Analysis' discusses a case (FindMedianSortedArrays) where optimization provides minimal improvement due to already-optimal O(log(min(m,n))) complexity, with detailed code examples in Appendix Figures 12-18."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 shows StarCoder2-15B MU increases by 5.0% after optimization. Table 3 shows Unsupervised Self-Refine and Result-Aware Self-Refine dramatically worsen performance (e.g., 518.8% TMU increase). Table 6 shows pass@1 decreases. GPT-4 MU and TMU improvements are modest."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about StarCoder2-15B (87.1% ET reduction from 0.93s to 0.12s, 90.8% TMU reduction from 22.02 to 2.03 Mb*s) and DeepSeek-6.7B-Ins (85.8% MU reduction) are exactly matched by Table 1."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The causal claim that overhead profiles cause efficiency improvement is supported by the ablation in Table 3, which compares EFFI-LEARNER against controlled variants (no feedback, result-only feedback, single-profiler feedback). This is adequate controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title ('Enhancing Efficiency of Generated Code') and abstract make broad claims about 'LLM-generated code' without bounding to Python. All experiments are Python-only. The limitation (Appendix A.1) acknowledges this but the title and abstract do not."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for why EFFI-LEARNER works. For example, the LLMs may be memorizing known efficient algorithms rather than using the profiling feedback, or the open test cases may be biasing optimization toward specific inputs."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper measures execution time and memory usage and claims improvements in execution time and memory usage. The claims match the granularity of measurements — no broader framing gap exists (they don't claim 'better code quality' or 'improved software engineering')."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Some models are versioned (GPT-3.5-Turbo-0301), but others use marketing names without versions: 'GPT-4' (no snapshot date), 'Claude-3-Sonnet' (no snapshot), 'GPT-4-Turbo' (no date). Section 4.2 says 'detailed versions are demonstrated in supplementary file' but the paper itself lacks this for key models."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Figure 3 and Appendix A.3 provide the full prompt template used in the self-optimization stage. The fill values (task description, test case, code, overhead analysis) are programmatically determined from known benchmarks and profiling output. Complete worked examples are shown in Appendix Figures 4-18."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The NeurIPS checklist (Q7) mentions 'greedy-decoding strategy' but the main paper does not specify temperature, top-p, max tokens, or other generation parameters. No hyperparameter table is provided."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The EFFI-LEARNER pipeline is described in detail in Section 3 with Figure 2: Code Generation → Overhead Profiling (line_profiler for time, memory_profiler for memory) → Code Refinement. The iterative loop, profiling tools, and feedback mechanism are clearly documented."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.2: 'We first collect the generated code from each LLM and evaluate its correctness using open test cases. Only the code that passes all test cases is considered for efficiency evaluation.' The filtering pipeline from generation to evaluation is clearly documented."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 contains a dedicated 'Limitations' section discussing three specific limitations: time-consuming multi-iteration process, increased token consumption, and Python-only evaluation."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 discusses specific threats: (1) the multi-iteration process is time-consuming for complex tasks, (2) overhead profiles consume additional tokens, (3) effectiveness is evaluated only on Python, so 'performance in different programming languages or environments may vary.'"
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Appendix A.1 explicitly states: 'the effectiveness of EffiLearner has been primarily evaluated on Python. Therefore, its performance in different programming languages or environments may vary, underscoring the need for further testing and validation.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "Only aggregated results (averages across benchmarks) are reported in tables. Individual per-problem efficiency measurements, profiling outputs, and generated code are not released for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1-4.2 describes the data collection: benchmarks used (EffiBench, HumanEval, MBPP), how code is generated (each LLM with greedy decoding), how profiling is performed (line_profiler, memory_profiler on open test cases), and how correctness is validated."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses standard public benchmarks (EffiBench, HumanEval, MBPP) as data sources."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: LLM generates code → correctness check on open test cases → profiling with line_profiler and memory_profiler → profile-guided refinement → repeat up to 5 iterations → final evaluation on private test cases. Filtering criteria are stated (only correct code evaluated)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Section 6 discloses funding: National Key R&D Program of China (2022ZD0160201), HK RGC RIF (R7030-22), HK ITF (GHP/169/20SZ), Huawei Flagship Research Grant 2023, HK RGC GRF (17208223 & 17204424), and HKU-CAS Joint Laboratory."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: University of Hong Kong, University of Edinburgh, Beijing University of Posts and Telecommunications, University College Dublin, University of Cambridge, King's College London, Shanghai AI Laboratory."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders include government grants and Huawei. The paper evaluates third-party models (GPT-4, Claude, StarCoder, CodeLlama, etc.) — none are Huawei products. The funders have no direct financial stake in which models perform better with EFFI-LEARNER."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial interest disclosure is present in the paper. The Huawei funding is acknowledged but there is no explicit declaration that authors have no competing interests."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 22 evaluated models. This is important since HumanEval (2021) and MBPP (2021) could be in the training data of newer models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether benchmark problems appeared in any model's training data. HumanEval and MBPP are widely known public benchmarks that could have been memorized by models trained after 2021."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (published 2021), MBPP (published 2021), and EffiBench (published 2024) are public benchmarks. Models trained after these dates may have seen the problems. No contamination analysis is performed."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. It is a benchmark evaluation of LLM code generation efficiency."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study evaluates LLMs on public coding benchmarks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The approach requires up to 5 iterative LLM calls per problem plus profiling, but no inference costs (API spend, tokens consumed, wall-clock time per problem) are reported. The limitations section acknowledges 'overhead profiles consume more tokens' but does not quantify it."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.2 describes the hardware (8× A100 GPUs, 128-core CPU) but does not state total GPU hours, API costs, or training/inference time for the full experimental campaign across 22 models and 3 benchmarks."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No seed sensitivity analysis is reported. The NeurIPS checklist claims greedy decoding makes results 'consistent,' but execution time and memory profiling measurements vary across runs, and no sensitivity analysis is performed."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of profiling runs per problem is not stated. It is unclear whether efficiency metrics are from a single profiling run or averaged across multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported for the prompt design, number of iterations, or profiling configuration. The iteration count (5) is explored in ablation but the prompt design process is undocumented."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 reports results at all optimization steps (0-5) rather than cherry-picking the best. The 5-iteration configuration is used consistently across all models, not selected per-model."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed, so multiple comparison correction is not applicable. The paper makes comparisons via raw number differences only."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement their own baselines (Unsupervised Self-Refine, Result-Aware Self-Refine) and compare against their own system without acknowledging author-evaluation bias. No independent evaluation is performed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "EFFI-LEARNER requires up to 5 additional LLM calls plus profiling per problem, substantially more compute than the baselines. This compute overhead is acknowledged qualitatively in limitations but never quantified or compared against the efficiency gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether EffiBench, HumanEval, or MBPP adequately measure real-world code efficiency. The benchmarks consist of small algorithmic problems — whether efficiency gains transfer to real-world codebases is not addressed."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The EFFI-LEARNER scaffold is applied uniformly across all models using the same prompt template (Figure 3), profiling tools, and iteration count. Cross-model comparisons use the same framework, controlling for scaffold effects."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval (2021) and MBPP (2021) were published years before most evaluated models were trained. Models may have memorized efficient solutions to these problems. No temporal leakage analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "During self-optimization, the LLM receives detailed per-line profiling data for specific test cases. This could lead to optimization overfitting to those specific inputs rather than general efficiency. This form of information leakage is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the open test cases used for optimization and the private test cases used for evaluation share structural similarities that could inflate results."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention methods are employed. No canary strings, membership inference, n-gram overlap analysis, or temporal splits are used."
       }
     }
   },
   "claims": [
     {
       "claim": "EFFI-LEARNER significantly improves LLM-generated code efficiency across diverse models, achieving up to 87.1% execution time reduction (StarCoder2-15B) and 90.8% total memory usage reduction.",
       "evidence": "Table 1 shows efficiency metrics before and after EFFI-LEARNER for 6 representative models on EffiBench. StarCoder2-15B ET: 0.93→0.12s, TMU: 22.02→2.03 Mb*s. Tables 5, 8, 9 extend to additional models and benchmarks.",
       "supported": "strong"
     },
     {
       "claim": "Overhead profile feedback is essential for effective code optimization; without it, naive self-refinement often degrades efficiency.",
       "evidence": "Table 3: For CodeLlama-70B, Unsupervised Self-Refine increases TMU by 518.8% and Result-Aware Self-Refine increases TMU by 523.2%, while EFFI-LEARNER reduces TMU by 92.9%. Similar pattern for GPT-3.5-Turbo-0301.",
       "supported": "strong"
     },
     {
       "claim": "Most efficiency improvement occurs in the first self-optimization step, with diminishing returns in subsequent steps.",
       "evidence": "Table 2: For CodeLlama-70B, MU drops 75.9% after step 1 and stays flat through step 5. For GPT-3.5-Turbo-0301, TMU drops 91.3% after step 1, reaching 92.1% by step 5.",
       "supported": "strong"
     },
     {
       "claim": "EFFI-LEARNER generalizes across benchmarks (EffiBench, HumanEval, MBPP) and diverse model families (16 open-source, 6 closed-source).",
       "evidence": "Tables 1, 4, 5, 8, 9 show efficiency improvements across all three benchmarks and 22 models. Improvements vary in magnitude but are consistently positive for most model-benchmark combinations.",
       "supported": "moderate"
     },
     {
       "claim": "EFFI-LEARNER's impact on code correctness is minimal, with pass@1 decreasing by only 0-0.5%.",
       "evidence": "Table 6 shows pass@1 before and after optimization on EffiBench for 16 open-source models. Maximum decrease is 0.5% (OpenCodeInterpreter-DS-1.3B: 5.8%→5.4%). Some models show no change (CodeLlama-7b: 7.0%→7.0%).",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or uncertainty quantification",
       "detail": "Efficiency measurements (execution time, memory usage) are inherently variable across runs, yet all results are reported as single point estimates with no variance, confidence intervals, or standard deviations. The justification that greedy decoding makes results 'consistent' applies only to code generation, not to runtime profiling."
     },
     {
       "flag": "Benchmark contamination unaddressed",
       "detail": "HumanEval and MBPP were published in 2021. Models trained after 2021 may have memorized efficient solutions to these problems. If models have seen canonical solutions, the 'self-optimization' may simply be recalling memorized efficient implementations rather than genuinely optimizing from profiles."
     },
     {
       "flag": "Selective reporting emphasis in abstract",
       "detail": "The abstract highlights StarCoder2-15B's 87.1% ET reduction — the single most dramatic result. Other models show much more modest improvements (e.g., GPT-4 ET only decreases 9.7%, and GPT-4 TMU only decreases 37.8%). The abstract's numbers represent best-case, not typical performance."
     },
     {
       "flag": "No statistical tests despite 'significant' claims",
       "detail": "The word 'significantly' is used repeatedly (e.g., 'EFFI-LEARNER significantly enhances the efficiency') but no statistical significance tests are performed. All comparisons are raw number differences across 22 models without any formal testing."
     },
     {
       "flag": "Inference cost of iterative approach not quantified",
       "detail": "EFFI-LEARNER calls the LLM up to 5 additional times per problem plus local profiling. For expensive closed-source models (GPT-4), this could multiply inference costs 6x. The paper acknowledges token overhead in limitations but never quantifies the cost-efficiency tradeoff."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces the HumanEval benchmark and Codex, foundational for LLM code generation evaluation."
     },
     {
       "title": "Program synthesis with large language models",
       "authors": ["Jacob Austin", "Augustus Odena", "Maxwell Nye"],
       "year": 2021,
       "arxiv_id": "2108.07732",
       "relevance": "Introduces the MBPP benchmark used to evaluate LLM code generation capabilities."
     },
     {
       "title": "EffiBench: Benchmarking the efficiency of automatically generated code",
       "authors": ["Dong Huang", "Jie M. Zhang", "Yuhao Qing", "Heming Cui"],
       "year": 2024,
       "arxiv_id": "2402.02037",
       "relevance": "Primary benchmark for evaluating code efficiency, with defined metrics (ET, NET, MU, NMU, TMU, NTMU) used throughout this paper."
     },
     {
       "title": "Self-Refine: Iterative Refinement with Self-Feedback",
       "authors": ["Aman Madaan", "Niket Tandon", "Prakhar Gupta"],
       "year": 2023,
       "relevance": "Key baseline for self-refinement approaches; demonstrates LLMs can improve outputs through iterative feedback loops."
     },
     {
       "title": "Teaching large language models to self-debug",
       "authors": ["Xinyun Chen", "Maxwell Lin", "Nathanael Schärli", "Denny Zhou"],
       "year": 2023,
       "arxiv_id": "2304.05128",
       "relevance": "Explores using program execution feedback to help LLMs debug code, related approach to EFFI-LEARNER's profile-guided optimization."
     },
     {
       "title": "Reflexion: Language agents with verbal reinforcement learning",
       "authors": ["Noah Shinn", "Federico Cassano", "Ashwin Gopinath"],
       "year": 2023,
       "relevance": "Demonstrates verbal self-reflection as feedback for improving LLM agent performance, conceptually related to EFFI-LEARNER's iterative self-optimization."
     },
     {
       "title": "Is your code generated by chatgpt really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2024,
       "relevance": "EvalPlus benchmark providing extended test cases for HumanEval and MBPP, used as private test sets in this paper's evaluation."
     },
     {
       "title": "Code Llama: Open foundation models for code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family (7B-70B) extensively evaluated in this paper as both open-source baseline and optimization target."
     },
     {
       "title": "StarCoder: May the source be with you!",
       "authors": ["Raymond Li", "Loubna Ben Allal", "Yangtian Zi"],
       "year": 2023,
       "arxiv_id": "2305.06161",
       "relevance": "Open-source code LLM trained on The Stack; StarCoder2-15B is the paper's headline result with 87.1% ET reduction."
     },
     {
       "title": "Learning Performance-Improving Code Edits",
       "authors": ["Alexander Shypula", "Aman Madaan", "Yimeng Zeng"],
       "year": 2024,
       "relevance": "PIE approach for learning code optimizations from performance-improving edits; used as a baseline in Table 7."
     },
     {
       "title": "ECCO: Can we improve model-generated code efficiency without sacrificing functional correctness?",
       "authors": ["Siddhant Waghjale", "Vishruth Veerendranath", "Zora Zhiruo Wang", "Daniel Fried"],
       "year": 2024,
       "arxiv_id": "2407.14044",
       "relevance": "Directly related work on improving LLM code efficiency while maintaining correctness."
     },
     {
       "title": "Evaluating language models for efficient code generation",
       "authors": ["Jiawei Liu", "Songrun Xie", "Junhao Wang"],
       "year": 2024,
       "arxiv_id": "2408.06450",
       "relevance": "Evaluates LLM code efficiency, finding generated code is less efficient than human solutions."
     }
   ]
 }