ai-research-survey

scan.json (21019B)

---

 {
   "paper": {
     "title": "Rethinking the Evaluation of Secure Code Generation",
     "authors": ["Shih-Chieh Dai", "Jun Xu", "Guanhong Tao"],
     "year": 2026,
     "venue": "ICSE 2026",
     "arxiv_id": "2503.15554",
     "doi": "10.1145/3744916.3773217"
   },
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The enhanced SecCodePLT+ dataset is released at https://github.com/Utah-SaLT-Lab/RethinkSecCodeEval (reference [9])."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The study uses two public datasets (BigCodeBench and SecCodePLT) and releases their enhanced SecCodePLT+ with unit tests at the GitHub repository."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup section is provided. The paper mentions specific model names and tools but not library versions or environment details."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. The GitHub link is given but no README with commands or reproduction guide is described in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results are reported as point estimates without confidence intervals or error bars. Figures show bar charts without error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": true,
         "justification": "T-tests are conducted to assess statistical significance between base models and secure code generation methods (Table 4), with p-values reported. Two-way ANOVA is also used (Section 4.3)."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Cohen's d is reported as effect size in Table 4, with interpretation guidance ('values around 0.2, 0.5, and 0.8 representing small, medium, and large effects')."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No power analysis or justification for why these particular dataset sizes or number of models were chosen. The manual inspection sample of 44 cases is described as using a fixed random seed but no justification for N=44."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper acknowledges not repeating every experiment due to resource limits. They repeated Qwen three times and state 'consistent results' but do not report standard deviations or variance across those runs."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Base models without secure code generation techniques serve as baselines throughout the study (shown as 'Base' in all figures)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The four evaluated techniques (SVEN, SafeCoder, CodeGuard+, PromSec) are described as 'state-of-the-art' methods from 2023-2024, including a CCS 2023 Distinguished Paper."
       },
       "ablation_study": {
         "applies": false,
         "answer": false,
         "justification": "This is an evaluation study of existing techniques, not a new system with components to ablate."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: Secure@1, Pass@1 (task-level and case-level), Secure-Pass@1, and the proposed SAFE@1."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Manual inspection is performed: 44 sampled cases for vulnerability scanner validation (Table 3), and manual classification of failure categories in Table 7 with disagreements resolved by majority voting among authors."
       },
       "held_out_test_set": {
         "applies": false,
         "answer": false,
         "justification": "This is not a training/learning study; the paper evaluates existing techniques on benchmark datasets."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down per model (5 open-source + 2 commercial), per scanner (CodeQL, Bearer, Bandit), per dataset (BigCodeBench, SecCodePLT+), per CWE type (Figure 5), and per failure category (Table 7)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Extensive failure analysis in RQ3 (Section 4.3) with five failure categories (Removed Code, Junk Code, NFI, FN, Other) and concrete examples in Figures 1, 2, and 9."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper's central finding is negative: existing secure code generation techniques often degrade functionality and show limited effectiveness when security and functionality are evaluated together."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about techniques compromising functionality (>50% degradation), CodeQL missing vulnerabilities, and 'garbage code' are all supported by results in Sections 4.1-4.4."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The paper makes causal claims about techniques causing functionality degradation. These are supported by controlled comparisons (same model with and without technique applied) and manual inspection of failure cases."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly bounds scope: Python-only, specific LLMs and scanners, and acknowledges tasks 'may not be representative.' The scope section (3.1) states focus on function-level code generation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper discusses alternative explanations: data contamination possibility, LLM-generated unit tests being potentially limited, static analyzer false negatives as confounds. Section 5 addresses multiple threats."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model names with sizes are given: CodeLlama-7B, Qwen2.5-Coder-7B, DeepSeek-Coder-V2-Lite, Mistral-7B, StarCoder-1B, GPT-3.5-Turbo, GPT-4o. However, exact API snapshot dates for GPT models are not given."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The paper describes prompts conceptually (e.g., 'we prompt Qwen2.5-Coder-32B to create a set of test inputs') but does not provide the actual prompt text used in experiments."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No temperature, top-p, or other sampling parameters are reported for the LLM experiments."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The four techniques' mechanisms are described in detail in Section 3.2 and Table 1 (fine-tuning, decoding control, prompt modification, external tool usage)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "The unit test generation process for SecCodePLT+ is documented (Section 3.2), including how ground truth code was used, pre-existing inputs were included, and indentation issues were corrected via Python syntax parser."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Limitations and Threats to Validity' provides a dedicated, substantive discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats discussed: static analyzer false negatives, Python-only evaluation, LLM-generated unit tests may be limited, manual calibration of problematic tests, resource constraints limiting repeated experiments (430 GPU-hours per full run)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explicitly states: 'we focus on function-level code generation,' 'does not aim at the performance of the most advanced models,' and Section 5 states 'these datasets focus mainly on Python, our findings may not generalize to other programming languages.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The SecCodePLT+ dataset with unit tests is released at the GitHub repository. BigCodeBench is a publicly available dataset."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "The unit test generation procedure for SecCodePLT+ is described in detail (Section 3.2): LLM-generated test inputs, ground truth code execution for expected outputs, average 7.5 test cases per task."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants were recruited. The study uses benchmark datasets and automated tools."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented: datasets selected, code generated by LLMs with/without techniques, scanned by three static analyzers, results aggregated. Unit test generation process for SecCodePLT+ is also documented."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Acknowledgments section lists NVIDIA Academic Grant Award, NSF awards #2340198, #2319880, #2213727, and Cisco University Research Program Fund #71858473."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All three authors are affiliated with University of Utah, clearly stated in the paper header. No evaluated product is from their institution."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": true,
         "justification": "Funders are NVIDIA, NSF, and Cisco. The paper evaluates open-source models and third-party techniques. None of the funders have a direct financial stake in the outcome of the evaluation."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state training data cutoff dates for any of the evaluated models."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses: 'the samples used in our evaluation might have been included in the training data of LLMs, potentially causing data contamination. Our employed two datasets are generally released after the evaluated models.'"
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The paper addresses this for SecurityEval (excluded because 'LLMs may have already been trained on these examples') and notes their datasets were 'generally released after the evaluated models' (Section 5)."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 states: 'a full run costs 430 GPU-hours (≈18 days) on a single NVIDIA A100' and the repeated Qwen experiment totaled '1,290 GPU-hours (≈54 days) on one A100.'"
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Total compute budget is stated: 430 GPU-hours per full run on NVIDIA A100, with 1,290 GPU-hours for repeated experiments."
       }
     }
   },
   "claims": [
     {
       "claim": "CodeQL fails to detect more than 20% of vulnerabilities in generated code",
       "evidence": "Table 3 shows CodeQL has 19 false negatives out of 44 manually inspected samples (43% FN rate). Section 4.1 and Figure 5 show CodeQL identifies far fewer CWEs than Bearer and Bandit.",
       "supported": "strong"
     },
     {
       "claim": "Existing secure code generation techniques often compromise functionality to enhance security, with many degrading base LLM performance by more than 50%",
       "evidence": "Figures 7-8 show SAFE@1 and Secure-Pass@1 scores. PromSec shows >50% drop on Qwen2.5-Coder and DeepSeek-Coder. Table 4 confirms statistical significance (p<0.05) with Cohen's d values.",
       "supported": "strong"
     },
     {
       "claim": "Techniques often remove vulnerable code entirely or generate garbage code rather than truly fixing vulnerabilities",
       "evidence": "Table 7 shows Removed Code accounts for 60-92% of failure cases, with concrete examples in Figures 1, 2, and 9.",
       "supported": "strong"
     },
     {
       "claim": "LLMs for vulnerability detection have high false positive rates (~50%) that hinder security assessment",
       "evidence": "Table 3 shows Qwen has 18/44 false positives and Llama has 21/44 false positives in manual validation.",
       "supported": "strong"
     },
     {
       "claim": "Security improvement by existing techniques is not monotonic; they may introduce new vulnerabilities into previously secure code",
       "evidence": "Table 6 shows 0.5-24.39% of originally secure code becomes insecure after applying techniques. Average 4.11% on BigCodeBench and 6.87% on SecCodePLT+.",
       "supported": "strong"
     }
   ],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "This ICSE 2026 paper demonstrates that existing secure code generation techniques (SVEN, SafeCoder, CodeGuard+, PromSec) show limited effectiveness when security and functionality are evaluated together on the same code. The techniques often sacrifice functionality to appear secure, primarily by removing vulnerability-related code or generating garbage output. CodeQL alone misses many vulnerabilities, and different static analyzers produce inconsistent results. The proposed SAFE metric and enhanced SecCodePLT+ dataset provide a more rigorous evaluation framework.",
   "red_flags": [
     {
       "flag": "Limited repeated experiments",
       "detail": "Due to resource constraints (430 GPU-hours per run), only the Qwen model experiments were repeated (3 times). All other results are single-run, though the authors acknowledge this limitation."
     },
     {
       "flag": "LLM-generated unit tests",
       "detail": "Unit tests for SecCodePLT+ were generated by Qwen2.5-Coder-32B rather than human-written. While manually calibrated, edge case coverage was not verified, which could affect functionality assessment accuracy."
     }
   ],
   "cited_papers": [
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "SVEN - one of the four secure code generation techniques evaluated in this study, CCS 2023 Distinguished Paper."
     },
     {
       "title": "Instruction Tuning for Secure Code Generation",
       "authors": ["Jingxuan He", "Mark Vero", "Gabriela Krasnopolska", "Martin Vechev"],
       "year": 2024,
       "relevance": "SafeCoder - fine-tuning-based secure code generation technique evaluated in this study."
     },
     {
       "title": "Constrained decoding for secure code generation",
       "authors": ["Yanjun Fu", "Ethan Baker", "Yu Ding", "Yizheng Chen"],
       "year": 2024,
       "arxiv_id": "2405.00218",
       "relevance": "CodeGuard+ - decoding-control-based secure code generation technique evaluated in this study."
     },
     {
       "title": "PromSec: Prompt Optimization for Secure Generation of Functional Source Code with Large Language Models (LLMs)",
       "authors": ["Mahmoud Nazzal", "Issa Khalil", "Abdallah Khreishah", "NhatHai Phan"],
       "year": 2024,
       "relevance": "PromSec - prompt-engineering-based secure code generation technique evaluated, CCS 2024."
     },
     {
       "title": "Bigcodebench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["Terry Yue Zhuo"],
       "year": 2024,
       "arxiv_id": "2406.15877",
       "relevance": "One of the two main evaluation datasets used in this study for code generation benchmarking."
     },
     {
       "title": "SecCodePLT: A Unified Platform for Evaluating the Security of Code GenAI",
       "authors": ["Yu Yang", "Yuzhou Nie", "Zhun Wang"],
       "year": 2024,
       "arxiv_id": "2410.11096",
       "relevance": "The other main dataset used, extended by the authors into SecCodePLT+ with unit tests."
     },
     {
       "title": "Purple llama cyberseceval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "CyberSecEval benchmark for LLM code security evaluation, discussed as related work."
     },
     {
       "title": "Understanding the effectiveness of large language models in detecting security vulnerabilities",
       "authors": ["Avishree Khare", "Saikat Dutta"],
       "year": 2025,
       "relevance": "Study on LLMs for vulnerability detection, findings referenced regarding LLM detection capabilities."
     },
     {
       "title": "Evaluating Large Language Models Trained on Code",
       "authors": ["Mark Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "HumanEval benchmark paper, foundational code generation evaluation benchmark discussed as insufficient for security assessment."
     },
     {
       "title": "CWEval: Outcome-driven Evaluation on Functionality and Security of LLM Code Generation",
       "authors": ["Jinjun Peng", "Leyi Cui"],
       "year": 2025,
       "arxiv_id": "2501.08200",
       "relevance": "Related benchmark that evaluates both functionality and security, discussed as limited by small dataset size."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan"],
       "year": 2022,
       "relevance": "Pioneering work on security assessment of LLM-generated code using CodeQL, established the single-scanner evaluation practice this paper critiques."
     },
     {
       "title": "SecurityEval dataset: mining vulnerability examples to evaluate machine learning-based code generation techniques",
       "authors": ["Mohammed Latif Siddiq", "Joanna CS Santos"],
       "year": 2022,
       "relevance": "Security evaluation dataset discussed as potentially contaminated since data comes from MITRE CWE web page."
     }
   ]
 }