ai-research-survey

scan.json (31370B)

---

 {
   "paper": {
     "title": "SafeGenBench: A Benchmark Framework for Security Vulnerability Detection in LLM-Generated Code",
     "authors": [
       "Xinghang Li",
       "Jingzhe Ding",
       "Chao Peng",
       "Bing Zhao",
       "Xiang Gao",
       "Hongwan Gao",
       "Xinchen Gu"
     ],
     "year": 2025,
     "venue": "arXiv",
     "arxiv_id": "2506.05692",
     "doi": "10.48550/arXiv.2506.05692"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SafeGenBench evaluates 13 LLMs on 558 security-related code generation tasks across 44 CWE types and 13 languages. Under zero-shot conditions, average security accuracy is only 37.44%, rising to 58.01% with safety instructions and 61.14% with few-shot examples. Reasoning models outperform non-reasoning models, with o3 and DeepSeek-R1 leading. Models perform best on memory safety violations (76.16%) and worst on insecure configurations (12.50%). The dual-judge framework (SAST + LLM) catches complementary vulnerability types, with the LLM-judge flagging 30% of issues SAST misses.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": false,
         "justification": "The abstract states 'The data and code will be released soon.' A promise of future release does not constitute actual release."
       },
       "data_released": {
         "applies": true,
         "answer": false,
         "justification": "Same as code — 'will be released soon' is stated but no working URL or archive is provided."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "No requirements.txt, Dockerfile, or environment setup section is provided. The paper does not describe dependencies or library versions needed to run the evaluation framework."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are included. The evaluation pipeline (code extraction, SAST scanning, LLM judging) is described conceptually but no runnable scripts or commands are provided."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Table 3 and Table 4 report only point estimates (e.g., '37.44%', '46.42%') with no confidence intervals, error bars, or ± notation. The only CI reported is for the LLM-Judge validation (95% CI 81.2%-96.8%), not for the main results."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'reasoning models consistently outperform non-reasoning models' and that safety instructions cause 'more than 20%' improvement, but no statistical significance tests (t-test, bootstrap, etc.) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper reports improvements with baseline context: 'average accuracy increases by more than 20%' (37.44% → 58.01% in Table 3), and per-category accuracy breakdowns in Table 4 provide absolute scores. The reader can compute effect magnitudes."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark contains 558 test cases but no justification is given for why this number is sufficient. No power analysis or sample size rationale is discussed."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "All results appear to be from single runs. No standard deviation, variance, or spread measures across experimental runs are reported in any table."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares SafeGenBench against three existing benchmarks (CodeLMSec, CWEval, SecRepoBench) on questions, CWEs, languages, scenario coverage, and evaluation method. 13 models are compared against each other."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "The evaluated models include very recent releases: o3-high, Gemini-2.5-Pro, Claude-3.7-Sonnet, DeepSeek-R1, Qwen3, Llama4-Maverick/Scout. Comparison benchmarks include CWEval (2025) and SecRepoBench (2025)."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 separately reports SAST-Judge and LLM-Judge scores alongside the combined overall score, effectively ablating the dual-judge evaluation framework. Section 6.4 analyzes each judge's contribution independently."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results are reported across Overall accuracy, SAST-Judge accuracy, and LLM-Judge accuracy (Table 3), plus per-category breakdowns across 8 vulnerability categories (Table 4), and per-CWE analysis (Figures 4-5)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": true,
         "justification": "Section 6.4 reports: 'A stratified random sample of 9% of the test cases was selected across all CWE categories. A security expert re-evaluated the cases blindly using standardized protocols. The LLM-Judge achieves 92% accuracy.'"
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "SafeGenBench is a newly created benchmark that models have not been trained or tuned on. The 558 test cases function as a held-out evaluation set with no model selection or tuning performed on them."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 provides per-category accuracy for all 13 models across 8 vulnerability categories. Figures 4-5 show per-CWE breakdowns for each judge type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B provides three detailed case studies (CWE-89 SQL injection, CWE-494 code integrity, CWE-798 hardcoded credentials) showing specific vulnerability patterns. Appendix C discusses divergent judge detections. Section 6.3 identifies worst-performing categories."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1 reports that SAST tools 'often suffer from limited detection effectiveness' and LLM judges 'performance degrades significantly when no vulnerability type is specified.' Section 6.4 quantifies complementary weaknesses: SAST misses 30% of vulnerabilities that LLM catches, while LLM misses 6.24% that SAST catches."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "The abstract claims 'notable deficiencies in their ability to produce vulnerability-free code' — supported by Table 3 showing 37.44% average zero-shot accuracy. The claim about the dual-judge framework is supported by Section 6.4's complementarity analysis."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims 'reasoning models consistently outperform non-reasoning models' (Section 6.1), attributing this to reasoning capability. This is an observational comparison across models that differ in many dimensions (size, training data, architecture), not controlled single-variable manipulation. No confound analysis is provided."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title and abstract frame results as generalizing to 'LLM-Generated Code' broadly, but test questions are written in Chinese (acknowledged only in the Limitations section), which confounds code security ability with Chinese language understanding. This significant scope constraint is not reflected in the title, abstract, or main body claims."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 6.3 speculates about training data distributions as an explanation for category-level variation but does not substantively consider alternatives: the Chinese language confound, model size effects, benchmark design bias, or whether CWE category difficulty varies intrinsically independent of training."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": false,
         "justification": "The paper measures vulnerability detection across 44 specific CWEs using a dual-judge framework, but frames the outcome as 'security of LLM-generated code' (abstract). The gap between 44 CWEs (out of thousands) and comprehensive code security is not discussed. The 8% error rate of the validated LLM-judge is not addressed as a measurement limitation."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are listed by marketing names only: 'GPT-4o', 'o1', 'o3-high', 'Gemini-2.5-pro', 'Claude-3.5-Sonnet', etc. No snapshot dates, API versions, or specific model identifiers (e.g., 'gpt-4o-2024-08-06') are provided."
       },
       "prompts_provided": {
         "applies": true,
         "answer": false,
         "justification": "The LLM-Judge system prompt is provided in Appendix D (Figure 6). However, the 558 test question prompts sent to the evaluated models are not provided — only three examples appear in Appendix B. The safety instruction and few-shot example prompts are described conceptually but not reproduced in full."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "No generation hyperparameters (temperature, top-p, max tokens) are reported for any of the 13 evaluated models or for DeepSeek-R1 used as the LLM judge."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. Models receive prompts directly and generate code responses."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the three-stage dataset construction pipeline: vulnerability type extraction/categorization (Stage 1), LLM-based test question generation (Stage 2), and human annotation/quality assurance (Stage 3). Section 4.2 describes the two-stage code extraction process from LLM outputs."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "A dedicated 'Limitations' section appears after Section 7 (Conclusion), discussing three specific limitations: single-function task difficulty, limited evaluation scope, and single-judge reliance."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations are specific to this study: (1) test cases limited to single-function generation, (2) no assessment of functional correctness, (3) single LLM judge and single SAST tool, (4) Chinese-language test questions may affect models differently based on their Chinese language capabilities."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "The Limitations section explicitly states what the results do NOT show: no project-level evaluation, no functional correctness assessment, and acknowledges the Chinese language constraint. These bound the scope of applicable conclusions."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "The raw data (558 test cases, model outputs, judge scores) is not available. 'The data and code will be released soon' is stated but no current access is provided."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 describes the three-stage data collection process: CWE taxonomy construction from OWASP Top-10 and CWE Top 25 (Stage 1), LLM-based test question generation with specific design principles (Stage 2), and expert review with acceptance/revision/rejection criteria (Stage 3)."
       },
       "recruitment_methods_described": {
         "applies": true,
         "answer": false,
         "justification": "Appendix A describes annotator demographics (6 experts, Asian, ages 24-35, 4 male/2 female, >3 years experience) but does not describe how these specific individuals were selected from the available pool. 'We employ a team of six human experts' does not explain the selection process."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": false,
         "justification": "The three construction stages are described conceptually, but filtering counts are missing: how many test questions were generated before human review? How many were revised vs. discarded? The paper jumps from the process description to '558 metriculously curated test cases' without intermediate counts."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding source is disclosed. All authors are affiliated with ByteDance, but no acknowledgments section lists grants or corporate funding for this research."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations with ByteDance are clearly listed in the paper header. Email addresses include bytedance.com domains."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "ByteDance is a commercial competitor to the companies whose models are evaluated (OpenAI, Google, Anthropic, Meta, Alibaba/Qwen). ByteDance develops its own AI products (MarsCode) that compete with the evaluated models, giving them a potential commercial interest in benchmark results showing competitor weaknesses."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is included in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "No training data cutoff dates are stated for any of the 13 evaluated models or for the DeepSeek-R1 judge model."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluated models may have seen similar CWE-specific coding scenarios during training. While the benchmark is newly created, the underlying vulnerability patterns and CWE examples are widely available online."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The benchmark uses well-known CWE patterns that exist extensively in training data. While the specific test questions are novel, the paper does not discuss whether models' familiarity with CWE-related coding patterns from training data affects results."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "The paper has no human participants as study subjects. The annotators are part of the research team constructing the benchmark, not study participants."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Appendix A mentions annotators were 'approved by their Ethics Review Committee' for the annotation work, but this is for the construction team, not study subjects."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study. Annotator demographics are reported in Appendix A for transparency, but these are research team members, not study subjects."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No API costs, tokens consumed, or inference time are reported for any of the 13 models evaluated across 558 test cases in 3 settings, or for the DeepSeek-R1 judge evaluations."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. The experiment involves 13 models × 558 test cases × 3 settings = 21,762 generation calls plus judge evaluations, but no cost or compute figures are reported."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of random seeds or seed sensitivity analysis. Results appear to be from single runs for all 13 models across all settings."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is not stated. It is unclear whether results are from single or multiple runs."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described or acknowledged. Generation parameters (temperature, sampling) are not reported, let alone any search over them."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The choice of DeepSeek-R1 as the unified LLM judge is not justified — no comparison with alternative judge models is provided. The selection of Semgrep as the SAST tool is not justified against alternatives."
       },
       "multiple_comparison_correction": {
         "applies": true,
         "answer": false,
         "justification": "The paper makes implicit comparisons across 13 models and 8 vulnerability categories but uses no statistical tests, let alone corrections for multiple comparisons."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors designed SafeGenBench and compare it favorably to prior benchmarks (Table 1) without acknowledging potential bias in this self-evaluation. No independent assessment of benchmark quality is conducted."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "The paper notes reasoning models outperform non-reasoning models but does not discuss the compute cost difference. Reasoning models (o3, DeepSeek-R1) use substantially more inference compute than standard models, but performance is not normalized by compute."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether SafeGenBench actually measures 'code security' as claimed, versus measuring familiarity with specific CWE patterns. The 44 CWEs cover a fraction of known weaknesses, and the construct validity of this selection as representative of 'security' is not examined."
       },
       "scaffold_confound_addressed": {
         "applies": false,
         "answer": false,
         "justification": "No scaffolding is involved. Models are queried directly with prompts."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The CWE-based vulnerability patterns used in test questions have existed online for years and are likely in all evaluated models' training data. While specific questions are novel, the underlying patterns are not."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Section 3.1.2 notes test questions deliberately avoid security terminology to prevent tipping off models, but this design choice is not discussed as a leakage concern. In the few-shot setting, explicit vulnerability examples are provided, which constitutes intentional feature provision but is not analyzed as a potential confound."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the 558 test cases are independent of each other or of model training data. Multiple test cases per CWE type may share structural similarity, but this is not addressed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. No canary strings, membership inference, or n-gram overlap analysis."
       }
     }
   },
   "claims": [
     {
       "claim": "Under zero-shot settings, the average overall accuracy across all 13 models is merely 37.44%, indicating substantial proportions of generated code contain vulnerabilities.",
       "evidence": "Table 3 shows per-model overall accuracy under zero-shot ranging from 27.78% (Llama4-Scout) to 46.42% (o3), averaging 37.44%.",
       "supported": "strong"
     },
     {
       "claim": "Explicit safety instructions in prompts increase average accuracy by more than 20 percentage points.",
       "evidence": "Table 3 shows average overall accuracy increasing from 37.44% (zero-shot) to 58.01% (zero-shot with safety instruction) across all 13 models.",
       "supported": "moderate"
     },
     {
       "claim": "Few-shot examples with insecure code improve accuracy by an additional 3 percentage points beyond safety instructions.",
       "evidence": "Table 3 shows average overall accuracy at 58.01% (zero-shot+SI) vs 61.14% (few-shot). Individual model improvements vary widely.",
       "supported": "moderate"
     },
     {
       "claim": "Reasoning models consistently outperform non-reasoning models across all three experimental settings.",
       "evidence": "Table 3 shows reasoning models (marked with *: Gemini-2.5-Pro, o1, o3, DeepSeek-R1, QWQ-32B) generally score higher, but overlap exists — e.g., o1 (35.30%) scores below Qwen3-MOE (41.22%) in zero-shot. No statistical test supports 'consistently.'",
       "supported": "weak"
     },
     {
       "claim": "Models perform best on Memory Safety Violations (76.16% average) and worst on Insecure Configurations (12.50% average).",
       "evidence": "Table 4 provides per-category accuracy across all 13 models under zero-shot. Memory Safety: 76.16% average; Insecure Configuration: 12.50% average; Resource Issues: 14.69% average.",
       "supported": "strong"
     },
     {
       "claim": "SAST and LLM-based judges exhibit complementary detection strengths — the LLM-Judge catches 30.05% of vulnerabilities missed by SAST.",
       "evidence": "Section 6.4 reports: 'The LLM-Judge flagged vulnerabilities missed by SAST in 30.05% of instances, while the SAST-Judge caught issues overlooked by the LLM in 6.24%. Only 2.59% were deemed vulnerable by both.'",
       "supported": "strong"
     },
     {
       "claim": "The LLM-Judge achieves 92% accuracy validated against expert ground truth.",
       "evidence": "Section 6.4: 'A stratified random sample of 9% of the test cases was selected... The LLM-Judge achieves 92% accuracy, with a 95% binomial confidence interval ranging from 81.2% to 96.8%.'",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Company evaluating competitors without conflict disclosure",
       "detail": "All authors are ByteDance employees. ByteDance develops competing AI products (MarsCode/Doubao). The paper evaluates only competitor models (OpenAI, Google, Anthropic, Meta, Alibaba, DeepSeek) without testing any ByteDance models and without a competing interests statement. The benchmark is designed to highlight security weaknesses in competitor products."
     },
     {
       "flag": "No error bars or multiple runs",
       "detail": "All results across 13 models × 3 settings appear to be single-run point estimates with no variance, confidence intervals, or reproducibility assessment. LLM generation is stochastic, so results could vary across runs."
     },
     {
       "flag": "Chinese-language prompts not disclosed prominently",
       "detail": "Test questions are written in Chinese, acknowledged only in the final paragraph of the Limitations section. This is a major confound — models' performance may reflect Chinese language ability rather than security awareness — yet the title and abstract frame results as general findings about LLM code security."
     },
     {
       "flag": "Unreleased artifacts prevent verification",
       "detail": "The 558 test cases, model outputs, and evaluation code are all unreleased ('will be released soon'). No reviewer or reader can currently verify any result."
     },
     {
       "flag": "Judge model choice not justified",
       "detail": "DeepSeek-R1 is used as the unified LLM judge with no justification for this choice over alternatives. If the judge has systematic biases, all LLM-Judge results are affected. The 92% validation accuracy has a wide 95% CI (81.2%-96.8%) from a small 9% sample."
     },
     {
       "flag": "Missing generation hyperparameters",
       "detail": "Temperature, top-p, and other sampling parameters are not reported for any of the 13 models or the judge. These settings significantly affect output variability and security patterns in generated code."
     }
   ],
   "cited_papers": [
     {
       "title": "Asleep at the keyboard? Assessing the security of GitHub Copilot's code contributions",
       "authors": ["Hammond Pearce", "Baleegh Ahmad", "Benjamin Tan", "Brendan Dolan-Gavitt", "Ramesh Karri"],
       "year": 2025,
       "relevance": "Foundational study on security vulnerabilities in LLM-generated code, directly motivating SafeGenBench's research question."
     },
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["Mark Chen", "Jerry Tworek"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Introduced HumanEval, the foundational code generation benchmark; contextualizes the shift from functional correctness to security evaluation."
     },
     {
       "title": "CodeLMSec Benchmark: Systematically Evaluating and Finding Security Vulnerabilities in Black-Box Code Language Models",
       "authors": ["Hossein Hajipour", "Keno Hassler", "Thorsten Holz", "Lea Schonherr", "Mario Fritz"],
       "year": 2024,
       "relevance": "Prior security benchmark for LLM-generated code using 280 insecure prompts; directly compared against in Table 1."
     },
     {
       "title": "CWEval: Outcome-driven evaluation on functionality and security of LLM code generation",
       "authors": ["Jinjun Peng", "Leyi Cui", "Kele Huang", "Junfeng Yang", "Baishakhi Ray"],
       "year": 2025,
       "arxiv_id": "2501.08200",
       "relevance": "Concurrent code security benchmark evaluating both functionality and security; directly compared against in Table 1."
     },
     {
       "title": "SecRepoBench: Benchmarking LLMs for Secure Code Generation in Real-World Repositories",
       "authors": ["Connor Dilgren", "Purva Chiniya", "Luke Griffith", "Yu Ding", "Yizheng Chen"],
       "year": 2025,
       "arxiv_id": "2504.21205",
       "relevance": "Concurrent benchmark for secure code generation in real-world repository contexts; directly compared against in Table 1."
     },
     {
       "title": "CYBERSECEVAL 2: A Wide-Ranging Cybersecurity Evaluation Suite for Large Language Models",
       "authors": ["Manish Bhatt", "Sahana Chennabasappa"],
       "year": 2024,
       "arxiv_id": "2404.13161",
       "relevance": "Meta's cybersecurity evaluation framework covering prompt injection and code security; contextualizes the behavioral vs code-level security distinction."
     },
     {
       "title": "LLMSecCode: Evaluating Large Language Models for Secure Coding",
       "authors": ["Anton Rydén", "Erik Näslund", "Elad Michael Schiller", "Magnus Almgren"],
       "year": 2024,
       "relevance": "Evaluation framework for secure code generation across multiple benchmarks; relevant prior work on evaluation methodology."
     },
     {
       "title": "GPT-4 technical report",
       "authors": ["Josh Achiam", "Steven Adler"],
       "year": 2023,
       "arxiv_id": "2303.08774",
       "relevance": "Technical report for GPT-4 family, one of the evaluated model families in SafeGenBench."
     },
     {
       "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning",
       "authors": ["Daya Guo", "Dejian Yang"],
       "year": 2025,
       "arxiv_id": "2501.12948",
       "relevance": "Technical report for DeepSeek-R1, both a top-performing evaluated model and the chosen LLM judge in SafeGenBench."
     },
     {
       "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
       "authors": ["Carlos E Jimenez", "John Yang", "Alexander Wettig"],
       "year": 2024,
       "relevance": "Key real-world code generation benchmark representing the shift from toy tasks to software engineering evaluation."
     },
     {
       "title": "Is your code generated by ChatGPT really correct? Rigorous evaluation of large language models for code generation",
       "authors": ["Jiawei Liu", "Chunqiu Steven Xia", "Yuyao Wang", "Lingming Zhang"],
       "year": 2023,
       "relevance": "EvalPlus benchmark augmenting HumanEval with adversarial tests; relevant to methodology of rigorous code evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code",
       "authors": ["Naman Jain", "King Han"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Addresses contamination in code benchmarks through temporal splits; relevant to benchmark design methodology."
     }
   ]
 }