ai-research-survey

scan.json (31607B)

---

 {
   "paper": {
     "title": "SecureAgentBench: Benchmarking Secure Code Generation under Realistic Vulnerability Scenarios",
     "authors": [
       "Junkai Chen",
       "Huihui Huang",
       "Yunbo Lyu",
       "Junwen An",
       "Jieke Shi",
       "Chengran Yang",
       "Ting Zhang",
       "Haoye Tian",
       "Yikun Li",
       "Zhenhao Li",
       "Xin Zhou",
       "Xing Hu",
       "David Lo"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2509.22097",
     "doi": "10.48550/arXiv.2509.22097"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SecureAgentBench evaluates 3 code agents (SWE-agent, OpenHands, Aider) with 3 LLMs on 105 real-world vulnerability-introduction tasks. The best combination (SWE-agent + DeepSeek-V3.1) achieves only 15.2% correct-and-secure solutions. Code agents introduce new vulnerability types not in the benchmark's historical records, and explicit security reminders in prompts do not improve secure coding outcomes. The backbone model has a stronger influence on performance than the agent framework, with DeepSeek-V3.1 roughly doubling the C&S rate of Claude 3.7 Sonnet and GPT-4.1.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "A repository URL is provided (https://github.com/iCSawyer/SecureAgentBench) directly under the abstract. However, the contributions section also states 'We will publicly release our code and dataset upon acceptance,' which creates ambiguity about current availability."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark tasks are derived from public OSS-Fuzz/ARVO vulnerabilities and a GitHub repository URL is provided. The 105 tasks with Dockerized environments are described as part of the release."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Each task includes a Dockerized environment (Section 2.1). Agent versions are specified: SWE-agent v1.1.0, OpenHands v0.50.0, Aider v0.86.1 (Appendix E.3). Hardware is listed: Intel Xeon Platinum 8480C, 2TB RAM, 8 NVIDIA H100 GPUs (Appendix E.1)."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E provides detailed experimental configurations for each agent. Docker environments are provided per task. Prompt templates are given in Appendix G (Figures 15-17). The GitHub repository would presumably contain reproduction scripts."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results are reported as point estimates (e.g., 15.2% C&S, 9.2% average). No confidence intervals, error bars, or ± notation appear in any table or figure. The authors acknowledge in limitations: 'we did not repeat experiments to fully mitigate the potential nondeterminism of LLMs.'"
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Claims like 'DeepSeek-V3.1 outperforms both Claude 3.7 Sonnet and GPT-4.1' (Section 3) are made by comparing raw percentages without any statistical significance testing."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported with sufficient baseline context for magnitude assessment: '14.3% vs. 7.3% and 6.0%' (Table 4), 'more than 80% of the correct code remains vulnerable' for CWE-120 vs 'only about 10%' for CWE-415 (Section 3). Cost comparisons also contextualize performance (Figure 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 105 tasks. The paper says this is 'comparable in scale to prior benchmarks (e.g., Peng et al. (2025))' but does not justify why 105 is sufficient for the claims made, nor is a power analysis provided."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Single-run results with no variance, standard deviation, or spread measures. The limitations explicitly state: 'Due to budget limitations, we did not repeat experiments to fully mitigate the potential nondeterminism of LLMs.'"
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Three agent frameworks (SWE-agent, OpenHands, Aider) and three LLMs (Claude 3.7 Sonnet, GPT-4.1, DeepSeek-V3.1) are compared against each other, providing a 3×3 matrix of comparisons (Table 4, Figure 4)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All models and agents are state-of-the-art at time of writing: Claude 3.7 Sonnet (Feb 2025), GPT-4.1 (Apr 2025), DeepSeek-V3.1 (Mar 2025), SWE-agent v1.1.0, OpenHands v0.50.0, Aider v0.86.1."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "The security reminder experiment (Figure 7) serves as an ablation, comparing performance with and without explicit security instructions using the best-performing setting (SWE-agent + DeepSeek-V3.1)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Six outcome categories are tracked: No Output, Compilation Error, Incorrect, Correct but Vulnerable, Correct but Suspicious, Correct and Secure (Section 2.1). Cost-effectiveness is also reported (Figure 5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Evaluation is entirely automated: functional test suites, PoC exploit execution, and SAST (Semgrep) scanning. No human evaluation of agent-generated code is performed. Manual inspection occurs only during benchmark construction (quality assurance), not evaluation of agent outputs."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The 105 tasks constitute a newly constructed benchmark not used for any tuning. Agents are evaluated with fixed configurations (temperature 0.0) without task-specific adaptation. The benchmark itself serves as a held-out test set for the evaluated agents."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 breaks down by agent and model. Figure 6 provides per-CWE vulnerability proportions. Table 5 shows CWE distribution of newly introduced vulnerabilities. Table 6 shows per-project distribution."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 discusses failure modes (invalid outputs, compilation errors). Figure 14 shows a concrete example of a correct-but-vulnerable patch. The security reminder analysis explains why more failures occur with the reminder. Appendix B discusses implications of failures."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The security reminder experiment (Figure 7) is a clear negative result: 'the number of securely resolved instances does not increase (i.e., 16 vs. 16), the cases yielding no valid output rise.' Aider's poor performance is also honestly reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claim of 15.2% C&S for best agent matches Figure 4 and Table 4. Claim about agents introducing new vulnerabilities is supported by Table 5. Claim about security reminders being insufficient is supported by Figure 7. All three key findings are substantiated in the results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The security reminder experiment is a controlled comparison (same agent, same model, prompt varied). Claims about model vs. agent influence are supported by the factorial design (3 agents × 3 models). The paper uses appropriately hedged language ('suggests', 'indicates') for most findings."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The limitations section explicitly bounds scope: 'SecureAgentBench is built on vulnerabilities identified by OSS-Fuzz; hence, the range of vulnerability types and programming languages may be somewhat constrained.' The benchmark is C/C++ specific and focuses on memory-safety vulnerabilities."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": true,
         "justification": "The paper offers alternative explanations for the security reminder result: 'the additional security reminder makes the agent more cautious, prompting extra deliberation and testing. This, however, increases the likelihood of hitting time and cost limits.' Limitations discuss SAST tool characteristics and LLM nondeterminism as confounds."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper directly measures functionality (test suite passing) and security (PoC exploit + SAST) and frames claims in terms of these same measurements. The 'Correct and Secure' metric is clearly defined. No significant gap exists between what is measured and what is claimed."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "Specific model versions are named: Claude 3.7 Sonnet, GPT-4.1, DeepSeek-V3.1 (DeepSeek-Chat). Agent versions are also specified: SWE-agent v1.1.0, OpenHands v0.50.0, Aider v0.86.1 (Appendix E.3). However, no API snapshot dates are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates are provided in Appendix G: Figure 15 (default), Figure 16 (with security reminder), and Figure 17 (requirement generation). The templates include placeholders for working_dir and problem_statement, and an example requirement is shown in Figure 11."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.3 reports: temperature 0.0 for all models, maximum 75 iterations and cost limit of 2 USD for SWE-agent and OpenHands. Aider temperature 0.0 with no explicit iteration/cost limits. Browser interaction disabled for OpenHands and Aider."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "Appendix E.2-E.3 describes each agent's architecture: SWE-agent uses terminal commands and bash-based tool execution; OpenHands uses CodeAct agent with browser disabled; Aider integrates with Git for iterative editing. Each agent's version and interaction mode is specified."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 and Figure 2 document the full pipeline with counts at each stage: 4,993 ARVO instances → 1,632 (single VIC) → 254 (validated) → 232 (oracles) → 105 (quality assured). Each filtering criterion is described in detail."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B contains a 'Limitations and Future Work' subsection with substantive discussion of three specific limitations: OSS-Fuzz scope constraints, Semgrep tool limitations, and lack of repeated experiments."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Specific threats are discussed: 'the range of vulnerability types and programming languages may be somewhat constrained' (OSS-Fuzz dependency), 'its findings may reflect the characteristics of its scanning rules and mechanisms' (Semgrep), and 'we did not repeat experiments to fully mitigate the potential nondeterminism of LLMs' (single runs)."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The paper states what it does NOT cover: other vulnerability sources beyond OSS-Fuzz, non-C/C++ languages, reasoning models ('We do not use reasoning models due to budget constraints'). It explicitly positions against BaxBench: 'we mainly focus on the evolution stage of software (i.e., code editing based on existing code bases).'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "A GitHub URL is provided but the paper states 'We will publicly release our code and dataset upon acceptance,' indicating the data is not yet publicly available. The benchmark tasks and evaluation data cannot be independently verified at time of publication."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes data collection in detail: vulnerabilities sourced from OSS-Fuzz via ARVO, VIC identification using a two-stage static+dynamic approach, PoC validation across three commits, and evaluation oracle acquisition. Each step has explicit criteria."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are public vulnerability databases (OSS-Fuzz, ARVO) and open-source repositories."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 2 shows the full pipeline with counts: 4,993 → 1,632 (single VIC candidate) → 254 (VIC validated) → 232 (oracles acquired) → 105 (quality assured). Each reduction step is explained with specific filtering criteria in Section 2.2."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding statement, acknowledgments section, or grant information appears in the paper. The authors are from six different universities but no funding sources are mentioned."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are clearly listed: Singapore Management University, National University of Singapore, Monash University, Aalto University, York University, Zhejiang University. None are affiliated with the companies whose products are evaluated (Anthropic, OpenAI, DeepSeek)."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Authors are at universities unaffiliated with the evaluated products, but the absence of any funding disclosure prevents verification."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present anywhere in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the three LLMs used (Claude 3.7 Sonnet, GPT-4.1, DeepSeek-V3.1). This is important because the vulnerability fixes and PoC programs are publicly available on GitHub and OSS-Fuzz."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The requirement generation process avoids leaking vulnerability details, but whether the LLMs' training data includes the vulnerability-fixing commits from GitHub is never discussed. The fixes are public and could be in training data."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The OSS-Fuzz vulnerabilities, fixing commits, and PoC programs are all publicly available on GitHub. LLMs trained on code could have seen the exact fixes. This contamination risk is not addressed anywhere in the paper."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The benchmark evaluates code agents on automated tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available vulnerability data and evaluates code agents."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 shows per-task cost for each agent-model combination. The text states DeepSeek-based agents cost 'below 0.2 USD per task' while SWE+GPT 'consumes more than 1.0 USD.' Cost limits of 2 USD per task are set for SWE-agent and OpenHands."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": true,
         "justification": "Hardware is specified: two Intel Xeon Platinum 8480C CPUs at 3.80 GHz, 2 TB RAM, 8 NVIDIA H100 GPUs with 80 GB HBM3 (Appendix E.1). Per-task cost limits are set at 2 USD. Per-task costs are shown in Figure 5."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper explicitly acknowledges: 'Due to budget limitations, we did not repeat experiments to fully mitigate the potential nondeterminism of LLMs.' All results are single-run, and temperature is set to 0.0 but output is still nondeterministic for these models."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not explicitly stated. The limitations imply single runs ('did not repeat experiments'), but this is not stated in the experimental setup or results sections."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. Temperature is fixed at 0.0 and iteration limits at 75, but no justification for these choices or exploration of alternatives is provided."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "All 9 agent-model configurations (3×3) are reported in Table 4 and Figure 4. The security reminder experiment uses the best-performing combination (SWE-agent + DeepSeek-V3.1) with explicit justification. No cherry-picking of configurations."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed in this paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors created the benchmark and evaluate third-party tools on it. While they don't implement the tools, potential bias in benchmark design (e.g., task selection favoring certain failure modes) is not discussed."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5 directly plots resolve rate (C&S) vs. cost per task for all 9 configurations. The paper explicitly discusses cost-effectiveness: 'Agents supported by DeepSeek-V3.1 are the most cost-effective, appearing in the upper-left area of the figure.'"
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2 extensively argues why the benchmark design measures secure code generation: aligned vulnerability introduction contexts, comprehensive evaluation combining functionality testing + PoC exploits + SAST. Table 1 compares construct dimensions against prior benchmarks. The three-part evaluation design is justified as more comprehensive than prior work."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The 3×3 factorial design (3 agents × 3 models) explicitly separates scaffold and model effects. Table 4 reports averages across agents and across models separately. The paper concludes 'it is the backbone model that exerts the stronger influence' based on this separation."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses historical OSS-Fuzz vulnerabilities with public fixing commits. The paper does not discuss whether the LLMs were trained on data that includes these fixes. The vulnerability fixes are publicly available on GitHub and could be in training data."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The requirement generation process (Section 2.2, Figure 17) explicitly ensures descriptions are 'security-neutral without explicitly mentioning vulnerabilities' and do not include 'code from the gold patches.' This prevents the prompt from leaking vulnerability or solution information."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Table 6 shows that harfbuzz alone accounts for 15.2% of tasks (16/105), and several projects contribute 6-7 tasks each. Non-independence of tasks from the same project (shared codebase, similar patterns) is not discussed as a potential confound."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection methods (canary strings, membership inference, n-gram overlap analysis) are applied. The paper only ensures security-neutral requirements, but does not detect whether training data contains the vulnerability fixes."
       }
     }
   },
   "claims": [
     {
       "claim": "Current code agents struggle to produce secure code, with the best combination (SWE-agent + DeepSeek-V3.1) achieving only 15.2% correct-and-secure solutions and an average of 9.2% across all configurations.",
       "evidence": "Figure 4 and Table 4 report C&S rates for all 9 agent-model configurations. SWE+DS achieves 15.2% and the overall average is 9.2%.",
       "supported": "strong"
     },
     {
       "claim": "Code agents introduce new types of security vulnerabilities not previously recorded in the benchmark, with 14 distinct CWE types detected by SAST compared to 11 historical types.",
       "evidence": "Table 5 reports CWE distribution of newly introduced suspected vulnerabilities, including CWE-14 (12.9%) which is not in the historical benchmark. Among correct solutions, 'more than 20% generated code is reported to produce new potential vulnerabilities' (Section 1).",
       "supported": "moderate"
     },
     {
       "claim": "An explicit security reminder does not significantly improve secure coding ability, with C&S remaining at 16/105 while invalid outputs increase.",
       "evidence": "Figure 7 compares SWE-agent + DeepSeek-V3.1 with and without security reminders. C&S stays at 16 while NO increases from 2 to 6 and CE from 28 to 31.",
       "supported": "moderate"
     },
     {
       "claim": "The backbone model has a stronger influence on performance than the agent framework, with DeepSeek-V3.1 achieving nearly twice the C&S rate of other models.",
       "evidence": "Table 4 shows model averages: DeepSeek 14.3% vs Claude 7.3% and GPT 6.0%. Figure 5 shows all three agent frameworks cluster similarly when using the same model. The paper states 'it is the backbone model that exerts the stronger influence.'",
       "supported": "moderate"
     },
     {
       "claim": "Claude 3.7 Sonnet generates the highest proportion of vulnerable outputs despite comparable functional correctness to DeepSeek-V3.1.",
       "evidence": "Table 4 reports Claude's functional correctness at 31.5% vs DeepSeek's 35.9%, but Claude's CV rate (17.5%) exceeds DeepSeek's (13.7%). The paper notes Claude's 'limited capability in ensuring software security.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No repeated experiments",
       "detail": "All results are single-run. The authors acknowledge 'we did not repeat experiments to fully mitigate the potential nondeterminism of LLMs.' With temperature set to 0.0, some stochasticity still exists in agent behavior, and conclusions drawn from single-run comparisons are less reliable."
     },
     {
       "flag": "No statistical tests for comparative claims",
       "detail": "Claims that DeepSeek-V3.1 'outperforms' other models and that agent frameworks differ in capability are based purely on comparing raw percentages from single runs. With 105 tasks, differences of a few percentage points could easily be within noise."
     },
     {
       "flag": "Contamination risk unaddressed",
       "detail": "The benchmark uses publicly available OSS-Fuzz vulnerability fixes from GitHub. LLMs trained on code repositories may have seen the exact fixing patches. The paper never discusses training cutoff dates or whether models could have memorized solutions."
     },
     {
       "flag": "Security reminder tested on single configuration only",
       "detail": "The security reminder experiment (Figure 7) is conducted only with SWE-agent + DeepSeek-V3.1. The claim that security reminders are insufficient is based on this single configuration, limiting generalizability to other agent-model combinations."
     },
     {
       "flag": "SAST false positive rate not quantified",
       "detail": "The 'Correct but Suspicious' category relies on Semgrep SAST, which the paper acknowledges 'may yield false positives.' The CS rate is reported as a security metric without quantifying false positive rates, potentially inflating perceived security risks."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R Narasimhan"],
       "year": 2024,
       "relevance": "Foundational coding agent benchmark that SecureAgentBench extends to security evaluation."
     },
     {
       "title": "SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering",
       "authors": ["John Yang", "Carlos E Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik R Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "One of three code agent frameworks evaluated in SecureAgentBench."
     },
     {
       "title": "OpenHands: An Open Platform for AI Software Developers as Generalist Agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "One of three code agent frameworks evaluated, representing extensible agent scaffold design."
     },
     {
       "title": "Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Prior secure coding benchmark for LLMs, compared against in Table 1."
     },
     {
       "title": "Asleep at the Keyboard? Assessing the Security of GitHub Copilot's Code Contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2025,
       "relevance": "Found ~40% of Copilot code completions are vulnerable, motivating secure code generation evaluation."
     },
     {
       "title": "BaxBench: Can LLMs Generate Correct and Secure Backends?",
       "authors": ["Mark Vero", "Niels Mündler", "Victor Chibotaru"],
       "year": 2025,
       "relevance": "Closest related benchmark combining correctness and security evaluation for LLM-generated code."
     },
     {
       "title": "CWEval: Outcome-Driven Evaluation on Functionality and Security of LLM Code Generation",
       "authors": ["Jinjun Peng", "Leyi Cui", "Kele Huang", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2025,
       "relevance": "Prior benchmark evaluating both correctness and security of LLM-generated code."
     },
     {
       "title": "SecRepoBench: Benchmarking LLMs for Secure Code Generation in Real-World Repositories",
       "authors": ["Connor Dilgren", "Purva Chiniya", "Luke Griffith", "Yu Ding", "Yizheng Chen"],
       "year": 2025,
       "arxiv_id": "2504.21205",
       "relevance": "Repository-level secure coding benchmark focusing on single-function vulnerability fixing."
     },
     {
       "title": "SafeGenBench: A Benchmark Framework for Security Vulnerability Detection in LLM-Generated Code",
       "authors": ["Xinghang Li", "Jingzhe Ding", "Chao Peng"],
       "year": 2025,
       "arxiv_id": "2506.05692",
       "relevance": "Security benchmark using SAST and LLM-judge for evaluating LLM-generated function security."
     },
     {
       "title": "SWT-bench: Testing and Validating Real-World Bug-Fixes with Code Agents",
       "authors": ["Niels Mündler", "Mark Müller", "Jingxuan He", "Martin Vechev"],
       "year": 2024,
       "relevance": "Agent evaluation benchmark for unit test generation, extending the SWE-bench paradigm."
     },
     {
       "title": "Instruction Tuning for Secure Code Generation",
       "authors": ["Jingxuan He", "Mark Vero", "Gabriela Krasnopolska", "Martin Vechev"],
       "year": 2024,
       "relevance": "SafeCoder technique combining security-aware finetuning for improving LLM code security."
     },
     {
       "title": "Evaluating Agent-Based Program Repair at Google",
       "authors": ["Pat Rondon", "Renyao Wei", "José Cambronero"],
       "year": 2025,
       "arxiv_id": "2501.07531",
       "relevance": "Industry evaluation of agent-based program repair, informs benchmark design decisions."
     }
   ]
 }