ai-research-survey

scan.json (25106B)

---

 {
   "paper": {
     "title": "COFFE: A Code Efficiency Benchmark for Code Generation",
     "authors": ["Yun Peng", "Jun Wan", "Yichen Li", "Xiaoxue Ren"],
     "year": 2025,
     "venue": "Proc. ACM Softw. Eng. (FSE)",
     "arxiv_id": "2502.02827",
     "doi": "10.1145/3715727"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Section 10 (Data Availability) states: 'The code and data of STGen and COFFE are available at https://github.com/JohnnyPeng18/Coffe.' A specific GitHub URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark data (COFFE) is released alongside the code at https://github.com/JohnnyPeng18/Coffe, as stated in Section 10. The benchmark is built upon publicly available benchmarks (HumanEval, MBPP, Code Contests, APPS)."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 4.5 mentions 'Linux machine with Ubuntu 20.04.4 LTS' and specific hardware (Intel Xeon Platinum 8358P CPU, 128 cores, 2 TB memory) along with libraries Coverage.py and Cirron, but no requirements.txt, Dockerfile, or detailed dependency versions are provided in the paper."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not include step-by-step reproduction instructions. While a GitHub repository is referenced, the paper itself does not provide commands to run or a 'Reproducing Results' section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "The main results in Table 6 report only point estimates (efficient@1, pass@1, speedup) without confidence intervals or error bars. Table 4 reports RSD but not confidence intervals on the main performance metrics."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., Finding 1-4 comparing LLMs, STGen vs. baselines) based solely on comparing numeric values without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage improvements with baseline context throughout. For example, STGen improves RSD by '43.10% and 32.08% on Llama3.1 and GPT-4o' (Section 5.2), and efficient@1 values are compared with pass@1 values with explicit deltas (Table 6 column 'Δ')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is given for why 20 code solutions are sampled per problem, why 12 measurement repetitions are used, or why the specific number of problems in each benchmark split was chosen. The sample of 20 solutions per problem for RSD evaluation is stated but not justified."
       },
       "variance_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative Standard Deviation (RSD) is reported extensively. Table 4 shows RSD for CPU instruction count vs. execution time stability. Table 5 shows RSD for distinguishability of test cases. The paper explicitly discusses variance in measurements."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three baselines are included for STGen evaluation: Instruction Prompting, Few-shot Prompting, and Generator-based Prompting (Section 4.3, Table 5). Original correctness test cases serve as a baseline for distinguishability evaluation."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The baselines are adapted from recent test case generation methods (Wang et al. 2024 [79], Ouédraogo et al. 2024 [66], Liu et al. 2024 [49]). The 14 evaluated LLMs include recent models like GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro, DeepSeek V2, and Llama3.1."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The comparison between STGen (with contracts) and baselines (without contracts) effectively serves as an ablation showing the contribution of contracts. The comparison of expression/generator test cases vs. raw test cases also demonstrates component contributions. Table 5 shows the impact of each format."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used throughout: accuracy, line coverage, RSD for STGen evaluation (Table 5); pass@1, efficient@1, and speedup for LLM evaluation (Table 6); RSD and Pearson correlation for CPU instruction count validation (Table 4)."
       },
       "human_evaluation": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark evaluation paper measuring automated code generation efficiency. Human evaluation of the system's outputs is not relevant — the evaluation is inherently automated (pass/fail on test cases and CPU instruction count measurements)."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark uses established test splits from existing benchmarks (HumanEval, MBPP sanitized, Code Contests, APPS). The evaluation of LLMs on COFFE uses these test problems. The stressful test cases generated by STGen are new and independent of LLM training data."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by function-level vs. file-level code generation throughout (Tables 5, 6). Per-model results are provided for all 14 LLMs. Results are also broken down by model size family (e.g., CodeLlama 7B/13B/34B)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses cases where LLMs generate code slower than ground truth (speedup < 1.0), the significant performance drop from pass@1 to efficient@1, and notes that Llama3-8B achieves 0% efficient@1 at file level. Section 7 discusses measurement errors and limitations of baselines."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative findings are reported: larger LLMs do not significantly outperform smaller ones in efficient code generation (Finding 4); most LLMs achieve speedup < 1.0 at file level; the few-shot baseline achieves only 65.17% accuracy at file level (Table 5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about STGen's 99% accuracy, 96% line coverage, CPU instruction count stability (1000x smaller RSD), and four findings from evaluating 14 LLMs are all supported by specific results in Tables 4, 5, and 6."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about contracts improving test case generation accuracy. This is supported by controlled comparison: STGen (with contracts) vs. baselines (without contracts) on the same problems using the same LLM (GPT-4o). The ablation-style design adequately supports these claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 explicitly states that results are limited to Python and 'the code generation performance of LLMs on other programming languages such as C++ and Java may be different.' The title refers to 'code generation' broadly but the threats section appropriately bounds this."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 (Threats to Validity) discusses specific alternative explanations: measurement errors from code optimization techniques (Sec 7.1), potential performance changes from modifying baselines, and the limitation to Python that could affect findings for other languages (Sec 7.2)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.4 specifies exact API engine names: 'gpt-3.5-turbo', 'gpt-4o', 'DeepSeek-V2-0628', 'DeepSeek-V2-0724', 'claude-3-5-sonnet-20240620', 'gemini-1.5-pro'. Open-source models are specified with sizes (e.g., CodeLlama 7B/13B/34B, Phi3 3.8B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes the prompting approach at a high level (e.g., 'STGen prompts LLMs with contract, verified generated test cases as demonstrations') but does not provide the actual prompt text used for STGen, contract generation, or code generation experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Temperature 0 is mentioned only for problem selection sampling, not for the main code generation evaluation. No temperature, top-p, or other sampling parameters are stated for the main LLM evaluation in RQ3. The threshold of 5 for Phase III is stated, but key LLM generation parameters are missing."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The STGen pipeline is described in detail across Sections 3.2.1-3.2.4: three phases (contract generation, test case generation, test case-contract validation), with iterative feedback, LLM judge checker, and the flow between phases is shown in Figure 2."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 documents the data pipeline in detail: starting benchmarks, problem validation (removing conflicting solutions, file operations, inconsistencies), problem selection (removing problems no LLM can solve), with exact counts at each stage (Table 1 shows Ori., Val., Sel. columns)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 7 'Threats to Validity' is a dedicated section with two subsections: 7.1 'Threats to Internal Validity' and 7.2 'Threats to External Validity', spanning approximately one full page."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.1 discusses specific threats: measurement errors from code optimization techniques, the lack of existing stressful test case generation baselines requiring modifications, and mitigation strategies (Docker isolation, 12 runs with outlier removal). Section 7.2 discusses Python-specific limitations."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 7.2 explicitly states: 'we mainly focus on the evaluation of Python code generation in this paper' and acknowledges results may differ for C++ and Java. Section 2 states the focus on function-level and file-level (not repo-level) code generation, explaining why repo-level is excluded."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The COFFE benchmark data is available at https://github.com/JohnnyPeng18/Coffe (Section 10), which should include the problems, test cases, and generated solutions that underlie the reported results."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the data collection in detail: problems come from HumanEval, MBPP (sanitized version), Code Contests, and APPS test splits. The selection criteria, validation process, and filtering steps are all documented with specific counts."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved. The data sources are standard public benchmarks (HumanEval, MBPP, Code Contests, APPS)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented in Section 3.1: original problems (Table 1 'Ori.' column) → validation removing conflicts/file operations (Table 1 'Val.' column) → selection removing unsolvable problems (Table 1 'Sel.' column). For APPS: 5000 → 3106 → 300 with exact removal counts (1894 conflicts, 2223 unsolvable)."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "The Acknowledgment section states: 'This work is supported by the National Nature Science Foundation of China (No. 62302437).'"
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly stated: The Chinese University of Hong Kong and The State Key Laboratory of Blockchain and Data Security, Zhejiang University. None of the authors appear to be affiliated with the LLM companies being evaluated."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "The funder is the National Nature Science Foundation of China, a government funding agency with no commercial stake in the performance of any specific LLM being evaluated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates 14 LLMs on benchmarks that could be in their training data (HumanEval published 2021, MBPP published 2021) but does not state training data cutoff dates for any of the models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses HumanEval and MBPP, which are well-known benchmarks published in 2021 and widely available online. No discussion of whether evaluated models may have seen these problems during training. This is particularly concerning since the paper builds on these benchmarks."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "HumanEval and MBPP have been publicly available since 2021, and Code Contests since 2022. All models evaluated were trained after these dates. The paper does not address the contamination risk at all, despite this being a well-known concern for these benchmarks."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants are involved in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper uses multiple API-based LLMs (GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro, DeepSeek V2) and generates 20 solutions per problem across 14 models but does not report API costs, tokens consumed, or wall-clock time for the evaluation."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "The hardware is described (Intel Xeon Platinum 8358P, 128 cores, 2 TB memory) but total computational budget — GPU hours, total API spend, or total experiment time — is not reported despite extensive API usage across 14 models."
       }
     }
   },
   "claims": [
     {
       "claim": "CPU instruction count is 1000x more stable than execution time for measuring code efficiency, with RSD of 0.003%-0.005% vs. 2.37%-5.65%.",
       "evidence": "Table 4 (Section 5.1) shows RSD comparisons across four benchmarks. CPU instruction count achieves RSD of 0.003-0.005% while execution time achieves 2.37-5.65%. Pearson correlation between the two is 0.96-1.00.",
       "supported": "strong"
     },
     {
       "claim": "STGen achieves ~99% accuracy in stressful test case generation, outperforming baselines by up to 51.77%.",
       "evidence": "Table 5 (Section 5.2) shows STGen accuracy of 98.64% (function-level) and 98.91% (file-level) vs. baselines ranging from 65.17% to 94.86%.",
       "supported": "strong"
     },
     {
       "claim": "Stressful test cases generated by STGen better distinguish code solution efficiency, improving RSD by up to 43.10% over correctness test cases.",
       "evidence": "Table 5 (Section 5.2) shows RSD improvement: function-level from 19.05% to 27.26% (Llama3.1) and from 21.35% to 28.20% (GPT-4o).",
       "supported": "strong"
     },
     {
       "claim": "Current LLMs' performance drops significantly from correct to efficient code generation, with best efficient@1 of 46.97% vs. best pass@1 of 79.90% at function level.",
       "evidence": "Table 6 (Section 5.3) shows the comparison across all 14 LLMs. DeepSeek V2 Coder achieves best efficient@1 of 46.97% with pass@1 of 79.90%. The delta column shows 31-100% performance drops.",
       "supported": "strong"
     },
     {
       "claim": "Larger LLMs do not significantly outperform smaller LLMs in efficient code generation despite performing better in correct code generation.",
       "evidence": "Section 5.3 Finding 4: CodeLlama-34b achieves efficient@1 of 40.37% close to Llama3.1-405b's 39.58% despite being 10x smaller. Similar patterns observed at file level.",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "COFFE is a benchmark for evaluating the time efficiency of LLM-generated code, using CPU instruction count (1000x more stable than execution time) and a novel efficient@k metric combining correctness and efficiency. STGen, a contract-guided stressful test case generation approach, achieves ~99% accuracy and better distinguishes code solution efficiency. Evaluation of 14 LLMs reveals that efficient code generation is significantly more challenging than correct code generation (best efficient@1 of 46.97% vs. best pass@1 of 79.90%), and larger model size does not proportionally improve code efficiency.",
   "red_flags": [
     {
       "flag": "No benchmark contamination discussion",
       "detail": "The benchmark builds on HumanEval (2021) and MBPP (2021), which are widely available online and likely in the training data of most evaluated models. The paper does not discuss this contamination risk at all. While the efficiency measurement adds a novel dimension, the correctness pass@1 results could be inflated by memorization."
     },
     {
       "flag": "Missing hyperparameters for main evaluation",
       "detail": "Temperature 0 is mentioned only for problem selection, not for the main RQ3 evaluation of 14 LLMs. Sampling parameters (temperature, top-p, max tokens) for the main code generation experiments are not reported, making results potentially non-reproducible."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "All comparative claims (Findings 1-4, STGen vs. baselines) are based on raw numeric comparisons without any statistical testing. The claim that larger models do not significantly outperform smaller ones (Finding 4) would particularly benefit from formal testing."
     },
     {
       "flag": "No inference cost reporting",
       "detail": "The evaluation involves generating 20 solutions per problem across 14 models (including 4 commercial API-based models) on 756 problems, plus STGen test generation costs, but no API costs or total compute time are reported."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen", "Jerry Tworek", "Heewoo Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and Codex, foundational for code generation evaluation."
     },
     {
       "title": "EffiBench: Benchmarking the Efficiency of Automatically Generated Code",
       "authors": ["Dong Huang", "Jie M. Zhang", "Yuhao Qing", "Heming Cui"],
       "year": 2024,
       "arxiv_id": "2402.02037",
       "relevance": "First benchmark for evaluating code efficiency of LLM-generated code; direct precursor to COFFE."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major benchmark for evaluating LLMs on real-world software engineering tasks."
     },
     {
       "title": "Evaluating Language Models for Efficient Code Generation",
       "authors": ["Jiawei Liu", "Songrun Xie", "Junhao Wang"],
       "year": 2024,
       "arxiv_id": "2408.06450",
       "relevance": "Directly evaluates LLM code efficiency using existing benchmarks; motivates the need for COFFE."
     },
     {
       "title": "Learning Performance-Improving Code Edits",
       "authors": ["Alexander G Shypula", "Aman Madaan", "Yimeng Zeng"],
       "year": 2024,
       "relevance": "Proposes methods for LLM-based performance optimization of code; uses execution time measurement that COFFE replaces with CPU instruction count."
     },
     {
       "title": "Is Your Code Generated by ChatGPT Really Correct? Rigorous Evaluation of Large Language Models for Code Generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "EvalPlus framework for rigorous LLM code generation evaluation; provides the MBPP+ benchmark used in COFFE."
     },
     {
       "title": "PerfCodeGen: Improving Performance of LLM Generated Code with Execution Feedback",
       "authors": ["Yun Peng", "Akhilesh Deepak Gotmare", "Michael Lyu"],
       "year": 2024,
       "arxiv_id": "2412.03578",
       "relevance": "Uses execution feedback to improve LLM code efficiency; by the same first author."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "Agentic approach to automated software engineering; relevant to evaluating LLM code generation capabilities."
     },
     {
       "title": "AgentCoder: Multi-Agent-based Code Generation with Iterative Testing and Optimisation",
       "authors": ["Dong Huang", "Qingwen Bu", "Jie M. Zhang"],
       "year": 2023,
       "arxiv_id": "2312.13010",
       "relevance": "Multi-agent framework for code generation with iterative testing, relevant to agentic AI programming."
     },
     {
       "title": "Code Llama: Open Foundation Models for Code",
       "authors": ["Baptiste Rozière", "Jonas Gehring", "Fabian Gloeckle"],
       "year": 2023,
       "arxiv_id": "2308.12950",
       "relevance": "Open-source code LLM family evaluated in COFFE; important baseline for code generation."
     },
     {
       "title": "DeepSeek-Coder: When the Large Language Model Meets Programming",
       "authors": ["Daya Guo", "Qihao Zhu", "Dejian Yang"],
       "year": 2024,
       "arxiv_id": "2401.14196",
       "relevance": "Code-specialized LLM achieving top performance on COFFE's efficient@1 metric."
     }
   ]
 }