ai-research-survey

scan.json (19476B)

---

 {
   "paper": {
     "title": "CWEVAL: Outcome-driven Evaluation on Functionality and Security of LLM Code Generation",
     "authors": ["Jinjun Peng", "Leyi Cui", "Kele Huang", "Junfeng Yang", "Baishakhi Ray"],
     "year": 2025,
     "venue": "",
     "arxiv_id": "",
     "doi": ""
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "GitHub repository provided: https://github.com/Co1lin/CWEval, explicitly mentioned in the contributions section."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "CWEVAL-BENCH is open-sourced as part of the GitHub repository, containing 119 security-critical coding tasks."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is described in the paper. The paper does not specify library versions or dependencies needed to run the benchmark."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The paper references the GitHub repository but does not include commands or a reproducing-results section."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates (e.g., func@10 = 60.28%) with no confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes comparative claims (e.g., larger models achieve higher func-sec@k) but provides no statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports percentage differences with baseline context, e.g., 'From func@10 to func-sec@10, the performance drops around 30%' and specific numbers like 'up to 9.8% on func-sec@10 for Claude 3.5 Haiku.'"
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark has 119 tasks and n=100 samples per task. No justification is given for why these numbers are sufficient, nor is a power analysis discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported across experimental runs. The unbiased pass@k estimator is used but no uncertainty in the estimates is shown."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "The paper compares against prior benchmarks (SecurityEval, CyberSecEval) qualitatively and evaluates multiple LLMs as baselines against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Models evaluated include GPT-4o, Claude 3.5 Sonnet/Haiku, Gemini 1.5 Pro/Flash, and Llama 3.1 variants — all contemporary at time of writing."
       },
       "ablation_study": {
         "applies": true,
         "answer": false,
         "justification": "No ablation study is conducted on the CWEVAL framework components (e.g., effect of different test oracle types, specification design choices)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Two metrics are used: func@k (functional correctness) and func-sec@k (functional correctness and security), at multiple k values."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The paper evaluates LLM-generated code quality but uses only automated test oracles. No human evaluation of the system's outputs is included. Human verification was used to create the benchmark itself but not to evaluate outputs."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is a benchmark paper — the entire benchmark is the test set. There is no tuning/dev split relevant here."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as aggregate scores across all 119 tasks. No per-CWE-type or per-language breakdown is provided in the main results."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses failure cases of prior benchmarks (SecurityEval, CyberSecEval) with specific examples in Figure 1 and Figure 2, and discusses SafeCoder's degradation on CWEVAL-BENCH."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "SafeCoder fine-tuning showed significant functionality degradation on CWEVAL-BENCH (Table II), which is a negative result. Security instruction prompting showed mixed results with some models showing slight func@k decreases."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims about a notable portion of functional but insecure code and inaccuracy of previous evaluations are supported by the results in Section V (30% gap between func@k and func-sec@k, SafeCoder degradation)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SafeCoder's degradation is 'possibly due to' the model learning to avoid security-sensitive code entirely. This is a causal claim supported only by speculation, not by controlled analysis."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title says 'LLM Code Generation' broadly, but the benchmark covers only 31 CWE types out of hundreds, and 5 languages. The paper does not bound its claims to these specific CWEs and languages."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations are discussed for the observed security gap. For example, the role of training data composition, prompt sensitivity, or model architecture differences is not explored."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Exact model versions are specified: gpt-4o-mini-2024-07-18, claude-3-5-haiku-20241022, gemini-1.5-flash-002, etc. (Section V-B)."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The security instruction prompt is partially quoted ('Your code should be secure and should NOT contain any vulnerability') but the full prompts including task specifications sent to models are not provided in the paper."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Temperature settings (0.2, 0.4, 0.6, 0.8) and number of samples (n=100) are reported in Section V-B."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. The evaluation is direct code generation from prompts."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark creation process is documented in Section IV with four steps: coding tasks design, specifications writing, test oracle development, and multilingual evolution."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "No dedicated limitations or threats-to-validity section exists. The conclusion briefly mentions future work but does not discuss limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No threats to validity are discussed anywhere in the paper."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what the results do NOT show or what settings are excluded. The limited task count is acknowledged but framed as a resource constraint rather than a scope boundary."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark and evaluation pipeline are open-sourced on GitHub, allowing independent verification of tasks, test oracles, and reference implementations."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section IV describes the four-step process for creating CWEVAL-BENCH: task design from CWE documentation, specification writing, test oracle development, and multilingual evolution."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The benchmark is created by the authors using CWE documentation."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline from CWE documentation to final benchmark tasks is documented in Section IV with four explicit steps, though exact counts at each filtering stage are not provided."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All authors are from Columbia University, clearly stated. No evaluated product is affiliated with the authors."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether CWEVAL-BENCH tasks or CWE documentation examples could have appeared in model training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark tasks are inspired by CWE documentation which is publicly available. No discussion of whether models may have seen similar or identical examples during training."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper generates 100 samples per task across multiple models and temperatures but does not report API costs, tokens consumed, or wall-clock time."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The paper mentions 'limited budget' for RQ.2/RQ.3 but does not quantify it."
       }
     }
   },
   "claims": [
     {
       "claim": "There is a significant performance gap (~30%) between functional correctness and combined functional-security pass rates across all evaluated LLMs.",
       "evidence": "Figure 3 shows func@10 vs func-sec@10 with drops around 30% across all models, maximum 35.79% for Gemini 1.5 Flash (Section V-C1).",
       "supported": "strong"
     },
     {
       "claim": "Larger models within the same family achieve higher func-sec@k scores.",
       "evidence": "Table I shows larger variants (GPT-4o, Claude 3.5 Sonnet, Gemini 1.5 Pro) outperform smaller ones on func-sec@k, though differences on func@k are small (Section V-C2).",
       "supported": "moderate"
     },
     {
       "claim": "Simple security instruction prompting improves func-sec@k for most LLMs with only slight decrease on func@k.",
       "evidence": "Table II shows improvements up to 9.8% on func-sec@10 for Claude 3.5 Haiku with security instruction (Section V-C3).",
       "supported": "moderate"
     },
     {
       "claim": "SafeCoder fine-tuning causes significant functionality degradation on CWEVAL-BENCH despite showing improvements on separate functionality/security benchmarks.",
       "evidence": "Table II shows CodeLlama-7b-hf func@1 drops from 30.77% to 15.35% and func-sec@1 from 13.76% to 7.37% with SafeCoder (Section V-C3).",
       "supported": "strong"
     },
     {
       "claim": "Previous benchmarks (SecurityEval, CyberSecEval) have unclear specifications and inaccurate evaluations due to reliance on static analysis.",
       "evidence": "Figure 1 shows false positive/negative examples from static analysis on SecurityEval samples. Qualitative analysis in Sections I and II (Section I, Figure 1).",
       "supported": "moderate"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "CWEVAL introduces an outcome-driven evaluation framework that simultaneously assesses both functionality and security of LLM-generated code using dynamic test oracles rather than static analysis. Evaluation of 9 LLMs on CWEVAL-BENCH reveals a consistent ~30% gap between functional correctness and combined functional-security pass rates. Simple security instruction prompting provides modest improvements, while SafeCoder fine-tuning causes significant functionality degradation, exposing a previously undetected alignment tax that separate functionality/security benchmarks miss.",
   "red_flags": [
     {
       "flag": "No per-category breakdown",
       "detail": "Results are reported only as aggregates across all 119 tasks. No breakdown by CWE type or programming language is provided, making it impossible to assess where models struggle most."
     },
     {
       "flag": "No contamination analysis",
       "detail": "Benchmark tasks are inspired by publicly available CWE documentation. Models may have seen similar examples during training, but this is not discussed."
     },
     {
       "flag": "No limitations section",
       "detail": "The paper lacks any discussion of limitations or threats to validity, which is a significant methodological omission."
     },
     {
       "flag": "No statistical uncertainty",
       "detail": "All results are point estimates with no confidence intervals, error bars, or significance tests despite making comparative claims between models."
     }
   ],
   "cited_papers": [
     {
       "title": "SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques",
       "authors": ["M. L. Siddiq", "J. C. Santos"],
       "year": 2022,
       "relevance": "Prior security evaluation benchmark for LLM code generation that CWEVAL aims to improve upon."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen", "J. Tworek", "H. Jun"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduces HumanEval benchmark and pass@k metric, foundational for LLM code generation evaluation."
     },
     {
       "title": "Is your code generated by chatgpt really correct? rigorous evaluation of large language models for code generation",
       "authors": ["J. Liu", "C. S. Xia", "Y. Wang", "L. Zhang"],
       "year": 2024,
       "relevance": "EvalPlus benchmark for rigorous LLM code generation evaluation, methodology referenced by CWEVAL."
     },
     {
       "title": "Purple llama cyberseceval: A secure coding benchmark for language models",
       "authors": ["M. Bhatt", "S. Chennabasappa"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "CyberSecEval benchmark that CWEVAL critiques for unclear specifications and noisy data."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["J. He", "M. Vechev"],
       "year": 2023,
       "relevance": "SVEN system for security-focused fine-tuning of LLMs, related approach to secure code generation."
     },
     {
       "title": "Asleep at the keyboard? assessing the security of github copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2022,
       "relevance": "Early empirical study of security vulnerabilities in LLM-generated code (GitHub Copilot)."
     },
     {
       "title": "Livecodebench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code benchmark, addresses data leakage concerns in LLM evaluation."
     },
     {
       "title": "Cyberseceval 2: A wide-ranging cybersecurity evaluation suite for large language models",
       "authors": ["M. Bhatt", "S. Chennabasappa"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "Extended version of CyberSecEval for broader security evaluation of LLMs."
     },
     {
       "title": "Instruction tuning for secure code generation",
       "authors": ["J. He", "M. Vero", "G. Krasnopolska", "M. Vechev"],
       "year": 2024,
       "arxiv_id": "2402.09497",
       "relevance": "Instruction tuning approach for improving security of LLM-generated code."
     }
   ]
 }