ai-research-survey

scan.json (31549B)

---

 {
   "paper": {
     "title": "SECODEPLT: A Unified Benchmark for Evaluating the Security Risks and Capabilities of Code Agents",
     "authors": [
       "Yuzhou Nie",
       "Zhun Wang",
       "Yu Yang",
       "Ruizhe Jiang",
       "Yuheng Tang",
       "Xander Davies",
       "Yarin Gal",
       "Bo Li",
       "Wenbo Guo",
       "Dawn Song"
     ],
     "year": 2024,
     "venue": "arXiv",
     "arxiv_id": "2410.11096"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SeCodePLT introduces a two-stage benchmark construction pipeline yielding 5.9k samples across 44 CWE categories in Python, C/C++, and Java, achieving nearly 100% security relevance compared to 68% for CYBERSECEVAL. Evaluations of six SOTA models reveal that C/C++ and Java are significantly harder than Python (>50% failure on C/C++ secure coding, <20% patch generation pass@5), reasoning models outperform non-reasoning ones, and security reminders consistently improve performance. Claude-3.7-Sonnet shows much stronger safety alignment than GPT-4o on cyberattack assistance tasks (90%+ refusal rate vs <10% on critical steps).",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/ucsb-mlsec/SecCodePLT, stated in the abstract footnote."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "Dataset released on HuggingFace at https://huggingface.co/datasets/secmlr/SecCodePLT, stated in the abstract footnote."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or detailed environment setup is described in the paper. Only the cyberattack environment setup in Appendix G.3.1 describes server configurations, not the experimental evaluation environment."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "While code and data are released, the paper does not include step-by-step reproduction instructions or describe how to replicate the main experimental results."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Results in Figure 5 and Table 2 are point estimates (percentages, pass@1, F1 scores) without confidence intervals or error bars."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "Comparative claims such as 'Claude-3.7-Sonnet and O4-Mini have the best performance' and 'Qwen2.5-Coder has the worst performance' are made by comparing raw numbers without any statistical significance tests."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper reports raw percentages and rates for each model but does not compute formal effect sizes or frame differences as relative improvements with standardized measures."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification for why 5.9k samples or specific seed counts (130 Python, 1k C/C++, 509 Java) were chosen. No power analysis is presented."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No standard deviations, variance across seeds, or spread measures are reported for the main experiments."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares SeCodePLT against eight existing benchmarks including CYBERSECEVAL, PrimeVul, SecLLMHolmes, SVEN, CodeLMSec, BaxBench, and AutoPatchBench on multiple dimensions."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Comparison benchmarks include recent work: BaxBench (2025), AutoPatchBench (2025), CYBERSECEVAL (2023-2024). Models evaluated include O4-Mini (2025) and Claude-3.7-Sonnet (2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Multiple controlled comparisons serve as ablations: with/without security reminder (Fig 5), with/without CWE policy (Table 2), different context levels (Table 5, Appendix J.2), and different prompting strategies (Table 4, Appendix J.1)."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: security relevance rate, prompt faithfulness, pass@1, pass@5, F1 score for CWE identification, rule-based test pass rate, dynamic safety test pass rate, and line coverage."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Model outputs are evaluated via automated dynamic testing and rule-based checks. The only human evaluation of outputs is a brief spot check: 'We manually inspected a subset of valid patches and found that nearly all were correct patches' (Section 4.4), which is not a systematic human evaluation."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "The benchmark is a novel test set that evaluated models were not trained or tuned on. The mutation pipeline explicitly creates novel samples to prevent memorization (Section 3.1)."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by CWE category (Figs 4, 7, 8), by programming language (Python, C/C++, Java in Fig 5 and Table 2), and by task type (secure coding, vulnerability detection, patch generation)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix J.3 provides a detailed error analysis of SOTA models (Claude-3.7-Sonnet and O4-Mini) with a specific code example showing where models fail. Appendix H shows Cursor failure examples."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Several negative results are reported: Manual CoT performs worse than default prompting (Table 4), removing context significantly degrades performance (Table 5), and all models show poor C/C++ patch generation (<20% even at pass@5)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims about broader coverage (Table 1 shows 44 CWEs vs competitors), higher data fidelity (Section 4.1 shows ~100% security relevance), greater scale (5.9k vs competitors), and revealing model strengths/weaknesses (Sections 4.2-4.4) are all supported by experimental results."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Causal claims are made through controlled ablation-like experiments: 'providing a security reminder can consistently improve the selected models' performance' is justified by with/without comparisons on the same models and data (Fig 5). Context retrieval experiments (Table 5) similarly use controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Code Agents' but only one agent (Cursor) is tested in an appendix; main experiments evaluate six standalone LLMs. The benchmark covers three languages and 44 CWEs, which is well-specified in the text, but the title overpromises relative to agent coverage."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not substantively discuss alternative explanations for observed results. For example, the finding that C/C++ is harder is attributed to real-world codebase complexity, but confounds like data source differences (manual Python vs. extracted C/C++), model training data composition, or task formatting differences are not explored."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper clearly defines what it measures (pass@1 on dynamic tests, F1 for CWE identification, rule-based detection rates) and frames claims at the matching level of granularity. Dynamic test passage is a direct measure of code security, and the paper acknowledges the measurement limitation: 'dynamic testing is by nature unsound' (Section 4.4)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Main experiments use marketing names: 'DeepSeek-R1', 'QwQ-32B', 'GPT-4o', 'O4-Mini', 'Claude-3.7-Sonnet', 'Qwen2.5-Coder-7B' without API snapshot dates or version identifiers. Judge experiments in Appendix C.3 do specify 'GPT-4o-2024-08-06' and 'Claude-3.5-Sonnet-20240620', but main evaluation models lack these details."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt templates and examples are provided: Appendix A shows complete data format/prompts for coding tasks, Appendix B shows mutation prompts, Appendix C shows judge prompts, and Appendix G.3.4 shows the full cyberattack instruction prompt. Figure 3 illustrates task prompts for all three evaluation tasks."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'We set a large temperature to enable more variations' for mutation (Section 3.3) without specifying the value. No temperature, top-p, or max token settings are reported for model evaluations. The maximum conversation round for cyberattack evaluation is 40 (Appendix G.1)."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The cyberattack evaluation in Appendix G uses a multi-turn agentic setup that is described in detail: goal provision, shell environment management, command execution feedback loop, shell switching protocol, and maximum conversation rounds (40). The full instruction prompt is provided in Appendix G.3.4."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3 documents the full data creation pipeline in detail: seed generation methodology per language, mutation procedures, dynamic validation, Levenshtein distance filtering (threshold 0.8), and sample counts at each stage (Figure 1)."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": false,
         "justification": "There is no dedicated limitations section. Section 5 'Conclusion and Future Work' lists future directions that implicitly acknowledge limitations, and Section 4.4 notes 'dynamic testing is by nature unsound,' but these are scattered mentions, not a substantive limitations discussion."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": false,
         "justification": "No systematic threats-to-validity discussion. Section 4.4 acknowledges 'dynamic testing is by nature unsound, even if a patch passes all of our test cases, there is still no guarantee that it is correct,' which is a specific threat, but it is isolated. No discussion of construct validity threats, data quality threats, or external validity threats."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do NOT show. Section 5 frames limitations as future work opportunities (extending to more languages, enriching PoC tests, repository-level evaluation) rather than explicitly bounding the current claims."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "Raw benchmark data is available on HuggingFace at https://huggingface.co/datasets/secmlr/SecCodePLT, and code is available on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 describes seed generation in detail for each language: Python (manual CWE analysis and scenario creation), C/C++ (Arvo dataset extraction with clangd context, GPT-4.1 task description generation, fuzzing for functionality tests), and Java (Juliet Test Suite with obfuscation and JavaParser processing)."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Data sources are CWE databases, the Arvo dataset, and the Juliet Test Suite — all publicly available benchmark/test collections."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Figure 1 documents the full pipeline with sample counts at each stage. Section 3.3 details mutation (3 text mutations × 3 code mutations = up to 10 per seed), validation via dynamic testing, and filtering via Levenshtein distance (threshold 0.8). Appendix I provides additional pipeline charts for C/C++ and Java."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources or acknowledgments section is present in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: UC Santa Barbara, UC Berkeley, VirtueAI, UIUC, UChicago, University of Oxford, UK AI Safety Institute."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, making independence impossible to assess. Some authors are affiliated with VirtueAI (a commercial entity), and the paper evaluates commercial models without disclosing potential conflicts."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present. Authors include affiliations with VirtueAI but no declaration of financial interests related to the findings."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the six evaluated models, despite this being crucial for assessing whether benchmark seeds (especially from public CWE databases and Arvo/Juliet datasets) could be in training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 states the mutation pipeline 'can also create unseen samples for LLMs, ensuring the model cannot rely on memorization to complete the tasks,' directly addressing potential train/test overlap through data novelty."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The two-stage pipeline with mutation (variable/function name changes, instruction rephrasing) and Levenshtein distance filtering (Section 3.3) is explicitly designed to create novel samples that prevent contamination from memorized training data."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. The paper evaluates LLMs on a benchmark."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study involves automated evaluation of LLMs on coding tasks."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants or experimental conditions requiring randomization."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, token counts, or latency measures are reported for the model evaluations despite querying six models across thousands of samples."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No computational budget, GPU hours, or total API spend is stated for benchmark construction or model evaluation."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "Results appear to be from single runs. No analysis of sensitivity to random seeds or stochastic variation in model outputs."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of runs for main experiments (secure coding, vulnerability detection) is not stated. The cyberattack evaluation specifies 50 experiments per category and 500 for end-to-end, but main experiments lack this detail."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. The paper tests three prompting strategies (Table 4) but does not describe any systematic hyperparameter tuning."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The default prompting strategy is used for main results without justification for why it was selected. Table 4 shows different strategies perform similarly, but no formal selection process is described."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "Many comparisons are made across 6 models, 3 languages, multiple CWEs, and 3 tasks without any correction for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors built SeCodePLT and evaluate other benchmarks (CYBERSECEVAL) as competitors. They do not acknowledge the inherent bias of evaluating their own benchmark's advantages."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Models of vastly different sizes are compared (Qwen2.5-Coder-7B vs. GPT-4o, DeepSeek-R1) without any discussion of compute budget differences or performance normalized by compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 evaluates benchmark construct validity through security relevance assessment (nearly 100%), prompt faithfulness evaluation, and test case coverage analysis (90.92% line coverage). Both metrics are validated with two different LLM judges showing consistency (Appendix C.3)."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "Main model comparisons use direct LLM queries without scaffolding. The Cursor evaluation in Appendix D tests it as a bundled product. No scaffold confound exists in the main comparisons."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss temporal leakage. Seed data from Arvo and Juliet Test Suite may predate model training. While mutations create variants, the underlying vulnerability patterns and solutions may be in training data. No temporal analysis is provided."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation setup leaks answer information. The security reminder/policy deliberately provides hints, but unintentional feature leakage through task formatting or context is not discussed."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether mutated samples from the same seed are independent enough, or whether seed data patterns overlap with models' training distributions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection method (canary strings, membership inference, n-gram overlap analysis) is used. The mutation pipeline changes surface features (variable names, instruction wording) but does not detect or prevent leakage of underlying vulnerability patterns."
       }
     }
   },
   "claims": [
     {
       "claim": "SeCodePLT achieves nearly 100% in both security relevance and instruction faithfulness, significantly outperforming CYBERSECEVAL's 67.81% security relevance rate.",
       "evidence": "Figure 4a shows nearly 100% security relevance across all CWEs for SeCodePLT vs. varied rates for CYBERSECEVAL (e.g., CWE-338: 4/30, CWE-798: 20/37). Section 4.1 reports CYBERSECEVAL overall rate of 67.81%.",
       "supported": "strong"
     },
     {
       "claim": "Python is consistently less challenging than C/C++ and Java across all three evaluation tasks and all models.",
       "evidence": "Figure 5 shows higher correct rates on Python (e.g., Claude: 66-80% Python vs. 24-30% C/C++). Table 2 shows higher vulnerability detection and patch generation rates for Python across all models.",
       "supported": "strong"
     },
     {
       "claim": "Large and reasoning models perform better than small non-reasoning models on security tasks.",
       "evidence": "Figure 5 and Table 2 consistently show Qwen2.5-Coder-7B performing worst while Claude-3.7-Sonnet and O4-Mini perform best. QwQ-32B (reasoning) outperforms Qwen2.5-Coder (non-reasoning) from the same family.",
       "supported": "moderate"
     },
     {
       "claim": "Security reminders consistently improve model performance on secure coding tasks across all languages.",
       "evidence": "Figure 5 shows hatched bars (with reminder) consistently higher than solid bars (without) for all models and languages.",
       "supported": "strong"
     },
     {
       "claim": "SOTA models still perform poorly on C/C++ and Java vulnerability detection and patch generation, with less than 20% pass@5 for C/C++ patching.",
       "evidence": "Table 2 shows C/C++ patch generation pass@5 rates: Claude 0.193, O4-Mini 0.129, DeepSeek-R1 0.091, QwQ-32B 0.113, GPT-4o 0.174, Qwen2.5-Coder 0.",
       "supported": "strong"
     },
     {
       "claim": "GPT-4o poses significantly higher risk than Claude in assisting cyberattack implementation, with Claude showing >90% refusal rate on critical attack steps.",
       "evidence": "Figure 13 shows for Initial Access: GPT-4o refusal 8% vs. Claude 94%; for C2 & Execution: GPT-4o refusal 10% vs. Claude 90%. Based on 50 experiments per category.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "No error bars or variance reporting",
       "detail": "All main experimental results (Fig 5, Table 2) are point estimates without confidence intervals, standard deviations, or any uncertainty quantification. Given stochastic model outputs, this undermines the reliability of comparative claims."
     },
     {
       "flag": "No statistical significance tests",
       "detail": "Comparative claims ('Claude-3.7-Sonnet and O4-Mini have the best performance,' 'Qwen2.5-Coder has the worst performance') are based solely on comparing raw numbers without any significance testing."
     },
     {
       "flag": "Missing funding and conflict disclosures",
       "detail": "No funding sources, competing interests, or financial interest declarations are provided. Authors include an affiliation with VirtueAI (a commercial entity), and the paper evaluates commercial LLMs without disclosing potential conflicts."
     },
     {
       "flag": "Unequal data provenance across languages",
       "detail": "Python seeds are manually handcrafted while C/C++ and Java seeds are extracted from existing databases (Arvo, Juliet). The difficulty difference attributed to 'language difficulty' may be substantially driven by this data source difference, which is acknowledged but not controlled for."
     },
     {
       "flag": "No dedicated limitations section",
       "detail": "The paper lacks a structured discussion of limitations, threats to validity, or scope boundaries. Limitations are scattered across sections or framed as future work."
     },
     {
       "flag": "Compute-unmatched model comparisons",
       "detail": "Models of vastly different sizes (Qwen2.5-Coder-7B vs. GPT-4o/DeepSeek-R1) are compared on the same tasks without discussing compute differences, making it unsurprising that larger models perform better."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2022,
       "relevance": "Foundational benchmark for evaluating security of AI-generated code, directly comparable to SeCodePLT."
     },
     {
       "title": "Purple llama cyberseceval: A secure coding benchmark for language models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa", "Cyrus Nikolaidis"],
       "year": 2023,
       "arxiv_id": "2312.04724",
       "relevance": "Primary baseline benchmark that SeCodePLT directly compares against and critiques for low data quality."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Seminal work on code generation LLM evaluation (Codex/HumanEval), foundational to code security evaluation."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world GitHub issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2023,
       "arxiv_id": "2310.06770",
       "relevance": "Major code agent benchmark that SeCodePLT explicitly contrasts with as not security-specific."
     },
     {
       "title": "LLMs cannot reliably identify and reason about security vulnerabilities (yet?)",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar"],
       "year": 2024,
       "relevance": "Comprehensive evaluation of LLM vulnerability detection capabilities, directly comparable benchmark (SecLLMHolmes)."
     },
     {
       "title": "Vulnerability detection with code language models: How far are we?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim"],
       "year": 2024,
       "arxiv_id": "2403.18624",
       "relevance": "Largest vulnerability detection benchmark (PrimeVul) that SeCodePLT compares against for data quality."
     },
     {
       "title": "BaxBench: Can LLMs generate correct and secure backends?",
       "authors": ["Mark Vero", "Niels Mündler"],
       "year": 2025,
       "arxiv_id": "2502.11844",
       "relevance": "Contemporary benchmark for evaluating LLM-generated backend security with dynamic testing."
     },
     {
       "title": "RedCode: Risky code execution and generation benchmark for code agents",
       "authors": ["Chengquan Guo", "Xun Liu", "Chulin Xie"],
       "year": 2024,
       "relevance": "Evaluates LLM risks in generating malware, complementary to SeCodePLT's security evaluation scope."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["Jingxuan He", "Martin Vechev"],
       "year": 2023,
       "relevance": "Proposes SVEN dataset for secure code generation evaluation, directly compared in Table 1."
     },
     {
       "title": "Cybench: A framework for evaluating cybersecurity capabilities and risk of language models",
       "authors": ["Andy K Zhang", "Neil Perry"],
       "year": 2024,
       "arxiv_id": "2408.08926",
       "relevance": "CTF benchmark for evaluating LLM cybersecurity capabilities, complementary to SeCodePLT's approach."
     },
     {
       "title": "DeepSeek-R1: Incentivizing reasoning capability in LLMs via reinforcement learning",
       "authors": ["DeepSeek-AI"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "One of the evaluated SOTA reasoning models, showing reasoning capability impacts on security tasks."
     },
     {
       "title": "CWEval: Outcome-driven evaluation on functionality and security of LLM code generation",
       "authors": ["Jinjun Peng", "Leyi Cui", "Kele Huang"],
       "year": 2025,
       "arxiv_id": "2501.08200",
       "relevance": "Contemporary secure code generation evaluation benchmark using outcome-driven metrics."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released benchmark with code and data on GitHub/HuggingFace that practitioners can use to evaluate code LLM security, though it requires significant setup effort."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Results largely confirm expected patterns (larger models are better, security coding is hard) rather than challenging conventional wisdom."
     },
     "fear_safety": {
       "score": 2,
       "justification": "Demonstrates that SOTA models generate insecure code at high rates and GPT-4o can assist cyberattacks with low refusal rate, raising concrete AI safety concerns."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Shows CYBERSECEVAL has only 68% security relevance, mildly critical of Meta's benchmark quality, but framed constructively."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code repository and HuggingFace dataset are publicly available for anyone to run evaluations."
     },
     "brand_recognition": {
       "score": 2,
       "justification": "Evaluates well-known models (GPT-4o, Claude, DeepSeek) and authored by researchers from prominent institutions (UC Berkeley, Oxford) including Dawn Song and Bo Li."
     }
   }
 }