ai-research-survey

scan.json (29873B)

---

 {
   "paper": {
     "title": "SecCodePRM: A Process Reward Model for Code Security",
     "authors": [
       "Weichen Yu",
       "Ravi Mangal",
       "Yinyi Luo",
       "Kai Hu",
       "Jingxuan He",
       "Corina S. Pasareanu",
       "Matt Fredrikson"
     ],
     "year": 2026,
     "venue": "arXiv",
     "arxiv_id": "2602.10418"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SecCodePRM is a process reward model that assigns step-level security scores to code trajectories for vulnerability detection and secure code generation. It reports large improvements over prior SOTA on vulnerability detection (72% vs 60% accuracy on SVEN, 96.8% vs 72.5% on PrimeVul) and achieves near-perfect partial-code VD (100% F1 on PreciseBugs). The model claims no safety-utility tradeoff in code generation, maintaining or improving functional correctness while enhancing security through inference-time scaling.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states 'Code is available at SecCodePRM' with what appears to be a hyperlinked repository reference. This indicates a code release, though the actual URL is embedded as a hyperlink in the PDF."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets are publicly available standard benchmarks: BigVul, SVEN, PrimeVul, ReposVul, PreciseBugs. The paper uses these without modification for evaluation."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "The paper mentions 'NVIDIA 4*80GB GPUs (e.g., A100 or H100)' and 'DeepSpeed ZeRO-2' but provides no requirements.txt, Dockerfile, or specific library versions. Hardware is described but the software environment is not specified in enough detail to recreate."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are provided in the paper. Training details are given (Table 8) but no commands, scripts, or README-level instructions for replicating the experiments."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "All results in Tables 1-5 and 7-10 report point estimates only. No confidence intervals, error bars, or ± notation appear anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims SecCodePRM 'outperforms' baselines across all settings based solely on comparing raw numbers. No statistical significance tests (p-values, t-tests, bootstrap tests) are reported for any comparison."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides baseline context for improvements: 'surpasses LLMxCPG... by a margin of 12%' (Section 5.1), '30% gap in accuracy compared to SOTA methods' on PrimeVul, '11.09% on func@1 and increase of 12.98% on safe@1' (Section 5.3). Baselines and absolute values are given."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "No justification is provided for the evaluation sample sizes. Benchmark sizes are inherited from the original datasets but never discussed in terms of statistical power or adequacy for the claims made."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No variance, standard deviation, or spread measures are reported across experimental runs. All results appear to be from single runs. Table 11 shows dataset statistics but not experimental variance."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Extensive baselines across all experiments: VulSim, VulBERTA, ReGVD, LLMxCPG for VD; CodeT5, CodeBERT, UniXcoder, StarCoder2, GPT-3.5/4 variants for PrimeVul; multiple LLMs for partial VD; QwenPRM for CG."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "Baselines include LLMxCPG (USENIX Security 2025), GPT-oss-120B, Llama-4-Scout, Qwen2.5 series, Gemma-3, and QwenPRM. These are contemporary and competitive."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Section B.2 and Table 6 provide ablation on reward design and aggregation methods (softmax+min, last pos+min/binary/ave, softmax+ave, softmax+binary). Figure 9 further ablates the choice of reward function r1 vs r2."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Multiple metrics are used: accuracy, F1, precision, recall for VD; pairwise metrics P-C/P-V/P-B/P-R for PrimeVul; SR@k for SVEN CG; func@k and safe@k for CWEval; pass@k for LiveCodeBench."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "No human evaluation of the model's outputs is included. All evaluation is automated through benchmark metrics. The paper motivates its approach by analogy to human expert VD patterns (Figure 2) but never has humans evaluate SecCodePRM's actual predictions."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "Table 11 shows explicit train/test splits for all datasets. Standard benchmark test splits are used (SVEN, PrimeVul, ReposVul, PreciseBugs). PrimeVul uses chronological splitting."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Table 9 provides per-benchmark breakdowns across SVEN, PrimeVul, ReposVul, and PreciseBugs. Table 1 reports on specific CWE types (CWE-125, CWE-190, CWE-416, CWE-476). Figure 5 shows per-length-bin accuracy."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section B.9 presents a case study showing correct detection, but no examples of where SecCodePRM fails are shown or discussed. The paper only demonstrates successes of its own method."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "Figure 7 honestly shows QwenPRM achieves slight advantages over SecCodePRM at SR@5. Table 5 shows SecCodePRM slightly underperforms QwenPRM on some LiveCodeBench configurations (e.g., QC2.5-7B k=1: 41.49 vs 43.44)."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of outperforming prior approaches in all three settings (full VD, partial VD, CG) are supported by Tables 1-4, Table 9, and Figures 7/10. The claim of 'preserving code functional correctness' is supported by LiveCodeBench results in Table 5."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "Claims like 'process-level supervision is more effective at capturing function-level semantics' are supported by the ablation study in Section B.2/Table 6 which systematically varies the reward design. The two-stage training (Section 5) is a controlled single-variable manipulation."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": true,
         "justification": "The Limitation section (Section A) explicitly bounds generalization: 'trained primarily on C, C++, and Python datasets... generalization to other programming languages and paradigms (e.g., functional languages, domain-specific languages) remains underexplored' and 'effectiveness on zero-day vulnerabilities or novel attack patterns outside the training distribution is uncertain.'"
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No alternative explanations for the performance gains are discussed. The paper attributes improvements to process-level supervision without considering confounds such as training data composition, base model choice, or the specific benchmarks used."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper's claims closely match its measurements: VD accuracy/F1 on established CWE benchmarks, SR@k for security rate, safe@k requiring both functional correctness and safety. No broader framing beyond what was measured (e.g., they don't claim to solve 'all code security')."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": true,
         "justification": "The primary model is specified as 'Qwen2.5-Coder-7B-Instruct' (Section 5). Baseline models are identified with specific sizes: QC2.5-7B, QC2.5-32B, QC3-30B, Llama4-17B, Codestral-22B, Gemma3-12B, GPT-oss-120B. The Qwen model name includes a specific version identifier."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Section B.6 provides the full prompt text used for baseline partial VD: 'Given the previous code ..., determine whether the current code ... is vulnerable or not. Reason step by step, and answer with Yes or No.' SecCodePRM itself is a fine-tuned classification model, not a prompting approach."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Table 8 lists all training hyperparameters: epochs (3+1), learning rate (1e-4 → 1e-6), optimizer (Adam, β1=0.9, β2=0.95), weight decay (0.1), batch size (8×NGPUs), max sequence length (128K), DeepSpeed ZeRO-2. Detection threshold 0.5 and scaling N values also stated."
       },
       "scaffolding_described": {
         "applies": false,
         "answer": false,
         "justification": "No agentic scaffolding is used. SecCodePRM is an end-to-end model with a classification head; CG uses simple Best-of-N ranking."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the full data pipeline: step segmentation by double newlines, compact/structural merging via heuristic patterns (suffix and prefix patterns), contrastive labeling via sequence alignment (Gestalt Pattern Matching), label propagation using AST analysis. Figure 4 visualizes the pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section A is titled 'Limitation' and provides substantive discussion of language generalization and zero-day vulnerability limitations."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "The limitations identify specific threats: 'trained primarily on C, C++, and Python datasets' with uncertain generalization to 'functional languages, domain-specific languages,' and 'effectiveness on zero-day vulnerabilities or novel attack patterns outside the training distribution is uncertain.' These are specific to this study."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section A explicitly states what the results do not show: generalization to other languages/paradigms and effectiveness on zero-day/novel attack patterns. The evaluation scope is bounded to 'function-level and repository-level benchmarks with known CWE categories.'"
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "All evaluation datasets (BigVul, SVEN, PrimeVul, ReposVul, PreciseBugs) are publicly available standard benchmarks. The underlying data can be independently obtained and verified."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 describes the data construction pipeline in detail, and Section 4.1 analyzes training dataset properties. Table 11 provides comprehensive statistics (example counts, token ratios, step ratios, vulnerability ratios) for all datasets."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. All data comes from standard public vulnerability benchmarks."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4 documents the full pipeline: raw code → step segmentation → filtering (no-op patterns) → structural merging (suffix/prefix patterns) → contrastive labeling (sequence alignment, Gestalt Pattern Matching) → label propagation (AST-based caller function labeling). Token length hacking bias is identified and BigVul/PrimeVul Unpaired excluded from training."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources are mentioned anywhere in the paper. No acknowledgments section is present in the provided text."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are clearly listed: Carnegie Mellon University, Colorado State University, University of California Berkeley. No commercial affiliations."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. Since funding_disclosed is NO, this question cannot be answered affirmatively."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper. Absence of disclosure is not the same as absence of conflict."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The base model is Qwen2.5-Coder-7B-Instruct but its training data cutoff is never stated. This is relevant because several evaluation benchmarks (SVEN 2023, PrimeVul 2023, PreciseBugs 2023) could be in Qwen's training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "No analysis of whether Qwen2.5-Coder's pretraining data includes any of the evaluation benchmarks. PrimeVul's chronological splitting (mentioned in Section 4.1) addresses the dataset's internal split but not pretrained model contamination."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "BigVul (2020), SVEN (2023), PrimeVul (2023), PreciseBugs (2023), and ReposVul (2024) were all published before Qwen2.5-Coder's likely training cutoff. No contamination analysis is performed despite this risk."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. All evaluation is automated on code benchmarks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses publicly available code vulnerability datasets."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "No inference cost or latency is reported despite the paper claiming suitability for 'real-time feedback' and 'dense, immediate reward signals.' For a method claiming real-time applicability, inference latency is critical and missing."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Hardware is mentioned ('NVIDIA 4*80GB GPUs') but no total training time, GPU hours, or computational budget is quantified. Only hyperparameters (epochs, learning rate) are given without wall-clock duration."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds or seed sensitivity analysis. All results appear to be from a single training run."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The number of experimental runs is never stated. For CG, N=10/20 candidates are sampled per example, but the number of independent training/evaluation runs is not specified."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search budget is reported. The hyperparameters in Table 8 appear to be fixed choices with no discussion of how they were selected or how many configurations were tried."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "While Table 6 ablates reward aggregation strategies, the selection of the main model's training hyperparameters is not justified. No validation set selection procedure is described for choosing the final model configuration."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed at all, so correction for multiple comparisons does not arise. The paper compares raw performance numbers without any hypothesis testing."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No acknowledgment of self-evaluation bias. The authors implement and evaluate their own system against baselines without discussing whether their baseline implementations or evaluation setup could systematically disadvantage competitors."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "While the paper notes SecCodePRM (7B) outperforms models 'nearly 17 times its size,' no explicit compute-matched comparison is provided. The inference compute differences between Best-of-N with different N values and direct classification are not quantified."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether the evaluation benchmarks (SVEN, PrimeVul, PreciseBugs, ReposVul) actually measure real-world vulnerability detection capability. The paper assumes benchmark performance equals security effectiveness without questioning construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "For CG experiments, the same base generator (QC2.5-7B or QC2.5-32B) is used across all PRM comparisons (QwenPRM vs SecCodePRM vs probability-based ranking), properly isolating the PRM's contribution. For VD, SecCodePRM is an end-to-end model without scaffolding."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of temporal leakage. The benchmarks (BigVul 2020, SVEN 2023, PrimeVul 2023) predate the Qwen2.5-Coder model, meaning solutions could exist in the pretraining data. PrimeVul's chronological split addresses within-dataset temporal issues but not pretraining contamination."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of feature leakage. The training data construction uses vulnerability patches (vulnerable vs fixed code pairs), and it is not discussed whether evaluation test samples share structural features with training examples."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "No discussion of whether training and test examples are independent. Multiple datasets draw from overlapping open-source repositories, and potential overlap between training data (e.g., SVEN train) and test benchmarks is not analyzed."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": false,
         "justification": "No concrete leakage detection or prevention method is applied. The token length hacking analysis (Section 4.1) addresses data quality bias but not test set leakage. No canary strings, membership inference, or decontamination pipelines are used."
       }
     }
   },
   "claims": [
     {
       "claim": "SecCodePRM outperforms prior SOTA methods on full-code vulnerability detection across multiple benchmarks.",
       "evidence": "Table 1: 72% accuracy and F1 on SVEN vs 60% for LLMxCPG. Table 2: 93.66% P-C on PrimeVul Paired vs 12.94% for GPT-4 CoT. Table 3: 96.83% accuracy vs 72.50% for LLMxCPG. Table 4: 0.59 F1 on PreciseBugs vs 0.48 for MSIVD.",
       "supported": "moderate"
     },
     {
       "claim": "SecCodePRM (7B) outperforms models up to 17x its size on partial-code vulnerability detection.",
       "evidence": "Table 9: SecCodePRM achieves 50.17 F1 on SVEN, 55.71 on PrimeVul, 23.79 on ReposVul, and 90.00 on PreciseBugs, exceeding all baselines including GPT-oss-120B across all benchmarks.",
       "supported": "moderate"
     },
     {
       "claim": "SecCodePRM improves secure code generation without sacrificing functional correctness (no safety-utility tradeoff).",
       "evidence": "Table 5: On LiveCodeBench, SecCodePRM maintains or slightly improves pass@k (55.19% vs 53.23% baseline for QC2.5-32B k=1). Table 7: On CWEval, func@1 increases by 11.09% and safe@1 by 12.98%.",
       "supported": "moderate"
     },
     {
       "claim": "Human experts can identify security flaws using only 40-60% of code tokens from the beginning.",
       "evidence": "Figure 2 box plot showing expert human VD patterns, based on analysis of vulnerability detection patterns measuring tokens required from the beginning of code.",
       "supported": "weak"
     },
     {
       "claim": "Process-level reward modeling provides a more effective inductive bias for security tasks than raw parameter scaling.",
       "evidence": "Section 5.2: SecCodePRM (7B) outperforms 120B models. Table 9 shows larger models (QC2.5-32B, QC3-30B) perform worse than SecCodePRM on partial VD F1.",
       "supported": "moderate"
     },
     {
       "claim": "SecCodePRM's accuracy is robust to increasing code length.",
       "evidence": "Figure 5 and Section B.1/Figure 8 show accuracy remains stable as token counts increase on PrimeVul and PreciseBugs.",
       "supported": "moderate"
     }
   ],
   "red_flags": [
     {
       "flag": "Suspiciously large performance gaps",
       "detail": "SecCodePRM achieves 96.83% accuracy on PrimeVul Paired vs 72.50% for SOTA LLMxCPG — a 24pp gap. On PrimeVul pairwise metrics, P-C jumps from 12.94% (GPT-4 CoT) to 93.66%, and both P-V and P-R are exactly 0%, meaning the model never makes certain types of errors. A 100% F1 on PreciseBugs partial VD (Table 9) is extraordinary. These results warrant skepticism without error bars or multiple-run verification."
     },
     {
       "flag": "No uncertainty quantification",
       "detail": "No error bars, confidence intervals, standard deviations, or multiple-run results are reported for any experiment. Given the extraordinary claims, this makes it impossible to assess result stability or statistical significance."
     },
     {
       "flag": "No contamination analysis",
       "detail": "The base model (Qwen2.5-Coder) was released in 2024, after all evaluation benchmarks were published (BigVul 2020, SVEN 2023, PrimeVul 2023, PreciseBugs 2023, ReposVul 2024). The pretraining data likely includes these benchmarks' code. No contamination analysis is performed."
     },
     {
       "flag": "Missing inference latency for real-time claims",
       "detail": "The paper repeatedly claims suitability for 'real-time feedback' and 'dense, immediate reward signals' during interactive coding, but never reports inference latency or throughput. Without timing data, the real-time applicability claim is unsubstantiated."
     }
   ],
   "cited_papers": [
     {
       "title": "Evaluating large language models trained on code",
       "authors": ["M. Chen"],
       "year": 2021,
       "arxiv_id": "2107.03374",
       "relevance": "Foundational paper on LLM code generation evaluation (HumanEval benchmark)."
     },
     {
       "title": "SWE-bench: Can language models resolve real-world github issues?",
       "authors": ["C. E. Jimenez", "J. Yang", "A. Wettig", "S. Yao", "K. Pei", "O. Press", "K. R. Narasimhan"],
       "year": 2024,
       "relevance": "Major real-world code generation benchmark evaluating LLM capability on GitHub issues."
     },
     {
       "title": "LLMs cannot reliably identify and reason about security vulnerabilities (yet?): A comprehensive evaluation, framework, and benchmarks",
       "authors": ["S. Ullah", "M. Han", "S. Pujar", "H. Pearce", "A. Coskun", "G. Stringhini"],
       "year": 2024,
       "relevance": "Comprehensive evaluation showing LLMs struggle with vulnerability detection, directly motivating SecCodePRM."
     },
     {
       "title": "Asleep at the keyboard? Assessing the security of github copilot's code contributions",
       "authors": ["H. Pearce", "B. Ahmad", "B. Tan", "B. Dolan-Gavitt", "R. Karri"],
       "year": 2025,
       "relevance": "Empirical assessment of Copilot-generated code security, a key prior work on LLM code safety."
     },
     {
       "title": "A comprehensive study of LLM secure code generation",
       "authors": ["S.-C. Dai", "J. Xu", "G. Tao"],
       "year": 2025,
       "arxiv_id": "2503.15554",
       "relevance": "Comprehensive study of LLM secure code generation, directly related to the paper's CG evaluation."
     },
     {
       "title": "Vulnerability detection with code language models: How far are we?",
       "authors": ["Y. Ding", "Y. Fu", "O. Ibrahim", "C. Sitawarin", "X. Chen", "B. Alomair", "D. Wagner", "B. Ray", "Y. Chen"],
       "year": 2024,
       "arxiv_id": "2403.18624",
       "relevance": "PrimeVul benchmark and evaluation framework used as a primary evaluation dataset in this paper."
     },
     {
       "title": "LLMxCPG: Context-Aware vulnerability detection through code property graph-guided large language models",
       "authors": ["A. Lekssays", "H. Mouhcine", "K. Tran", "T. Yu", "I. Khalil"],
       "year": 2025,
       "relevance": "SOTA vulnerability detection baseline using LLM+static analysis hybrid approach, outperformed by SecCodePRM."
     },
     {
       "title": "Large language models for code: Security hardening and adversarial testing",
       "authors": ["J. He", "M. Vechev"],
       "year": 2023,
       "relevance": "SVEN benchmark and security hardening approach used for both training data and evaluation."
     },
     {
       "title": "LiveCodeBench: Holistic and contamination free evaluation of large language models for code",
       "authors": ["N. Jain", "K. Han", "A. Gu", "W.-D. Li", "F. Yan", "T. Zhang", "S. Wang", "A. Solar-Lezama", "K. Sen", "I. Stoica"],
       "year": 2024,
       "arxiv_id": "2403.07974",
       "relevance": "Contamination-free code generation benchmark used to verify no safety-utility tradeoff."
     },
     {
       "title": "CWEval: Outcome-driven evaluation on functionality and security of LLM code generation",
       "authors": ["J. Peng", "L. Cui", "K. Huang", "J. Yang", "B. Ray"],
       "year": 2025,
       "relevance": "Joint functionality and security code generation benchmark requiring both correctness and vulnerability-free code."
     },
     {
       "title": "BigCodeBench: Benchmarking code generation with diverse function calls and complex instructions",
       "authors": ["T. Y. Zhuo", "V. M. Chien", "J. Chim"],
       "year": 2025,
       "relevance": "Major code generation benchmark referenced in the LLM code generation evaluation landscape."
     },
     {
       "title": "Can you really trust code copilots? Evaluating large language models from a code security perspective",
       "authors": ["Y. Mou", "X. Deng", "Y. Luo", "S. Zhang", "W. Ye"],
       "year": 2025,
       "arxiv_id": "2505.10494",
       "relevance": "Evaluation of LLM code copilot security, directly relevant to AI-assisted coding safety."
     }
   ]
 }