ai-research-survey

scan.json (32830B)

---

 {
   "paper": {
     "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks",
     "authors": [
       "Hwiwon Lee",
       "Ziqi Zhang",
       "Hanxiao Lu",
       "Lingming Zhang"
     ],
     "year": 2025,
     "venue": "NeurIPS 2025",
     "arxiv_id": "2506.11791",
     "doi": "10.48550/arXiv.2506.11791"
   },
   "scan_version": 3,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval"],
   "key_findings": "SEC-bench introduces a multi-agent framework (SECVERIFIER) that automatically verifies 200 real-world CVE instances from 898 candidates at $0.87/instance, an 85.7% improvement over single-agent baselines. State-of-the-art code agents (SWE-agent, OpenHands, Aider) achieve at most 18.0% success on PoC generation and 34.0% on vulnerability patching, far below the >60% resolve rates seen on general software engineering benchmarks like SWE-bench. Failure analysis reveals distinct failure modes: compilation errors and lingering vulnerabilities dominate SWE-agent failures, while improper patch formatting dominates OpenHands failures.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "Code is released at https://github.com/SEC-bench/SEC-bench and dataset at https://hf.co/datasets/SEC-bench/SEC-bench, with a leaderboard at https://sec-bench.github.io. These URLs are provided on page 1."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The dataset is released on HuggingFace at https://hf.co/datasets/SEC-bench/SEC-bench. Docker images for each verified instance are also provided as part of the framework."
       },
       "environment_specified": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 specifies agent versions (SWE-agent 1.0.1, OpenHands 0.33.0, Aider 0.82.0), Docker container configurations, and the evaluation environment setup. Each benchmark instance is packaged as a Docker image with all dependencies."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": true,
         "justification": "The paper provides full prompt templates (Appendix B.4, B.5), agent configurations (Section B.3), Docker-based evaluation environments, and the released code repository includes the full framework for reproducing the benchmark and evaluations."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Main results in Table 4 are reported as point estimates without confidence intervals or error bars. One stability check reports std dev (30.0% ± 7.9% for SWE-agent + o3-mini over 5 runs), but this covers only one of the nine agent-model configurations."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The Wilcoxon signed-rank test (p=0.27) is used for the contamination analysis (Section 3.2), but no significance tests are applied to the main comparative claims between agents and models in Table 4. Claims like 'SWE-agent achieves 33.8% vs Aider at 20.0%' are based on raw number comparisons."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Effect sizes are reported with baseline context throughout: '85.7% improvement over CODEACT' (26.0% vs 14.0%), success rates broken down by component (Builder 81.7%, Exploiter 39.4%, Fixer 69.2%), and percentage-point differences in contamination analysis (Table 5)."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The 200 verified instances result from the automated pipeline, not from a pre-specified target. The 80-instance subset for agent comparison is motivated by 'budget constraints' (Section 3.1) with no power analysis or justification that 80 instances provide sufficient statistical power."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "Only one of nine agent-model configurations was repeated (SWE-agent + o3-mini, 5 runs, 30.0% ± 7.9%). All main results in Table 4 are single-run point estimates with no variance measures."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "SECVERIFIER is compared against CODEACT (single-agent baseline) in Table 3. For agent evaluation, three agent scaffolds (SWE-agent, OpenHands, Aider) are compared against each other across three models."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are contemporary: Claude 3.7 Sonnet, GPT-4o, o3-mini for models; SWE-agent v1.0.1, OpenHands v0.33.0, Aider v0.82.0 for agent scaffolds. CODEACT is built on the same OpenHands framework for controlled comparison."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 3 presents an ablation comparing multi-agent SECVERIFIER vs single-agent CODEACT on 50 random instances, isolating the impact of the multi-agent decomposition. Per-component success rates (Builder, Exploiter, Fixer) further decompose the pipeline."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "Results report resolved rate (%), submitted rate (%), average cost ($), and average steps. Failure analysis (Section 3.3) adds four failure type categories. Per-component success rates are also reported."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "Agent outputs are evaluated entirely by automated sanitizer verdicts. The manual verification in Section 2.5 is for benchmark construction quality (inspecting bug reports and patches), not for evaluating agent performance on the security tasks."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": false,
         "justification": "The 80-instance subset used for model/agent comparison (Table 4) is drawn from the same 200-instance dataset used for full evaluation. Claude 3.7 Sonnet was selected as the best model on the 80-instance subset, then evaluated on the full 200 instances that include those same 80."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Results are broken down by project (Table 1, 29 projects), by agent scaffold and model (Table 4), by task type (PoC generation vs patching), by failure category (Figure 2: NP, IF, CE, SV), and by pre/post-cutoff (Table 5)."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.3 provides detailed failure analysis with four categories (No Patch, Improper Format, Compilation Error, Still Vulnerable). Specific failure examples are given, such as OpenHands modifying ~7,000 lines in gpac.cve-2023-0358 and SWE-agent misidentifying root cause in mruby.cve-2022-1201."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The overall low success rates (18.0% PoC, 34.0% patching) are themselves negative results showing current LLM limitations. Aider's consistently poor performance is reported. The exploiter agent's low from-scratch PoC generation rate (3/289 in Appendix E) is also reported."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims are supported: $0.87/instance matches Table 1 average; 18.0% PoC and 34.0% patching match full-dataset evaluation in Section 3.2; the 85.7% improvement over CODEACT matches Table 3 (26.0% vs 14.0%)."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The main causal claim—that the multi-agent framework improves verification—is supported by a controlled comparison against CODEACT using the same underlying framework (OpenHands), same 50 random instances, isolating the multi-agent variable (Table 3)."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title claims 'Real-World Software Security Tasks' and abstract says 'software security tasks' generally, but the benchmark only covers C/C++ memory safety vulnerabilities detectable by sanitizers. While Section 5 acknowledges the C/C++ focus, the title and abstract overstate the scope relative to the actual evaluation."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper attributes low agent performance to inherent task difficulty without discussing alternative explanations: whether prompt design, cost limits ($1.0-$1.5), iteration caps (75), or the evaluation setup itself contribute to low scores. The contamination analysis partially addresses one alternative but the main results lack such discussion."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly justifies sanitizer verdicts as the evaluation oracle (Section 2.1, Appendix B.2), discussing why this provides 'objective, deterministic verification without subjective judgment.' Section 5 acknowledges this only covers memory safety vulnerabilities, not all security engineering capabilities."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Models are identified by marketing names only: 'Claude 3.7 Sonnet', 'GPT-4o', 'o3-mini', 'Claude 3 Haiku'. No snapshot dates, API version identifiers, or model build dates are provided for any model."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Full prompt text is provided in Appendix B.4 (PoC generation, Figure 4), B.5 (vulnerability patching, Figure 5), and Appendix D (all SECVERIFIER agent prompts: Builder D.1, Exploiter D.2, Fixer D.3, single-agent D.4)."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section B.3 reports temperature (0.0 for all models), maximum iterations (75 for SWE-agent/OpenHands), and cost limits ($1.5 for Claude 3.7 Sonnet, $1.0 for GPT-4o and o3-mini). Agent versions are also specified."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "SECVERIFIER's multi-agent architecture is described in detail (Section 2.3): Manager, Builder, Exploiter, Fixer agents with specific roles, coordination logic, and iterative refinement. Evaluation agent configurations are described in Section B.3 (SWE-agent terminal interface, OpenHands CODEACT scaffold, Aider Git integration)."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 documents the full pipeline with counts: 38,201 OSV instances → 4,836 with sufficient documentation → 898 with sanitizer reports → 200 verified instances. Each filtering step's criteria are described. Section 2.5 documents the manual inspection process."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 'Limitations and Future Work' provides substantive discussion of two specific limitations: the C/C++ language restriction and the memory safety sanitizer subset of vulnerability types."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 discusses specific threats: 'we focus on C/C++ projects due to the reliability of memory safety sanitizers,' 'our current implementation covers a specific subset of vulnerability types detectable by memory safety sanitizers.' These are specific to this study's design choices."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": true,
         "justification": "Section 5 explicitly states what is not covered: other programming languages, vulnerability types beyond memory safety, web applications, operating system kernels, and distributed systems. Future work directions to address these gaps are described."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": true,
         "justification": "The full dataset is released on HuggingFace (https://hf.co/datasets/SEC-bench/SEC-bench) including verified instances with Docker images, PoC artifacts, and patches. The source code for the framework is on GitHub."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.2 describes collection from the OSV database, customized web scraping of bug reports from diverse platforms (GitHub Issues, RedHat Bugzilla, Chromium Issue Tracker), and adaptation of OSS-FUZZ configurations. Inclusion criteria at each stage are specified."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard public vulnerability databases (OSV, NVD) and associated bug tracking platforms."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The pipeline is documented with counts at each stage: 38,201 initial OSV instances → 4,836 with documentation → 898 candidates with sanitizer reports → 200 verified instances across 29 projects. Manual verification (Section 2.5) documents three rounds of inspection with specific criteria."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding acknowledgments or grant information is provided anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "Author affiliations are listed: University of Illinois Urbana-Champaign and Purdue University. The paper evaluates third-party models and agents, not products from the authors' institutions."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. The absence of a funding disclosure is itself the issue."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests statement or financial disclosure is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "Training cutoffs are stated only for the contamination analysis models: GPT-4o (KC: Sep 2023) and Claude 3 Haiku (KC: Aug 2023) in Section 3.2. The main evaluation model (Claude 3.7 Sonnet) and o3-mini do not have their training cutoffs stated."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 presents a dedicated data contamination analysis comparing 15 pre-cutoff and 15 post-cutoff instances based on CVE reserved dates, with Wilcoxon signed-rank test (p=0.27) showing no significant difference."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": true,
         "justification": "The contamination analysis (Table 5) uses CVE reserved dates to split instances relative to model knowledge cutoffs and tests for performance differences. The Wilcoxon test finds no significant contamination effect. However, this analysis covers only GPT-4o and Claude 3 Haiku, not the main evaluation models."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study. The benchmark evaluates automated LLM agents on security tasks."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. The study uses public vulnerability databases and automated agent evaluation."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in the evaluation. The manual verification involves two authors but this is quality assurance, not a human subjects study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": true,
         "justification": "Per-instance costs are reported throughout: $0.87 average for SECVERIFIER (Table 1), per-agent average costs in Table 4 (e.g., SWE-agent + Claude 3.7 Sonnet: $1.29 for patching, $1.52 for PoC), and SECVERIFIER vs CODEACT costs in Table 3."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "Per-instance costs and average steps are reported, but no total computational budget (total API spend, total GPU hours, or aggregate cost for the full evaluation campaign) is stated."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 reports a stability analysis: 'we select SWE-agent and o3-mini as the representative agent and model, and repeat the experiments five times. The average success rate is 30.0% with a standard deviation of 7.9%.' However, this covers only one of nine configurations."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The stability check explicitly states 5 runs for one configuration (SWE-agent + o3-mini). The main results in Table 4 do not state the number of runs, implying single-run evaluations."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "No hyperparameter search is described. Agent configurations (temperature, iteration limits, cost caps) appear to be set without any reported search process or justification for the specific values chosen."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.1 explains: 'we evaluate the best-performing agent on the full dataset' and 'The reason to select Claude 3.7 Sonnet is that it has better performance than other models in our evaluation over a random selected 80-instance subset.'"
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "Only one formal statistical test is performed (Wilcoxon signed-rank for contamination). Main agent comparisons use raw percentages without statistical tests, so no multiple comparison correction is applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors compare their SECVERIFIER framework against CODEACT without acknowledging self-evaluation bias. No independent evaluation or discussion of author-implementation bias in the baseline comparison."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "Cost limits differ by model ($1.5 for Claude 3.7 Sonnet vs $1.0 for GPT-4o and o3-mini) but performance is not analyzed as a function of compute budget. No performance-vs-cost curves are provided."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.1 and Appendix B.2 discuss why sanitizer verdicts are appropriate evaluation oracles. Section 5 explicitly acknowledges that the benchmark covers only memory safety vulnerabilities in C/C++, not all security engineering. Comparison with SWE-bench statistics (Table 2) contextualizes the benchmark."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Table 4 presents a full 3×3 grid of agent scaffolds × models, allowing separation of scaffold effects from model effects. Section 3.2 analyzes both dimensions independently, noting that SWE-agent and OpenHands achieve comparable performance while Aider is consistently lower."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 3.2 uses CVE reserved dates to split instances into pre/post knowledge-cutoff groups, directly addressing temporal leakage. The Wilcoxon test (p=0.27) finds no significant temporal effect."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 2.5 addresses feature leakage through manual inspection of bug reports: 'We inspect all bug reports and remove directly provided patches while preserving essential context.' This prevents agents from simply copying patch code from the task description."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Multiple instances come from the same project (e.g., 43 from gpac, 31 from imagemagick, 21 from mruby). The paper does not discuss whether performance on instances from the same project is correlated or whether this violates independence assumptions in the evaluation."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "A concrete detection method is applied: temporal splitting of 30 instances by CVE reserved date relative to model knowledge cutoffs, with Wilcoxon signed-rank test for statistical comparison (Table 5, p=0.27)."
       }
     }
   },
   "claims": [
     {
       "claim": "SECVERIFIER successfully verifies 200 real-world CVE instances, an 85.7% improvement over the single-agent CODEACT baseline.",
       "evidence": "Table 1 shows 200 verified from 898 seed instances (22.3% overall). Table 3 compares SECVERIFIER (26.0%) vs CODEACT (14.0%) on 50 random instances across 23 projects.",
       "supported": "moderate"
     },
     {
       "claim": "State-of-the-art code agents achieve at most 18.0% success in PoC generation and 34.0% in vulnerability patching.",
       "evidence": "Section 3.2 reports full-dataset (200 instances) evaluation results. Table 4 shows detailed 80-instance results across 3 agents × 3 models. Stability check on one config shows 30.0% ± 7.9% over 5 runs.",
       "supported": "moderate"
     },
     {
       "claim": "SEC-bench automatically creates benchmark instances at $0.87 per verified instance.",
       "evidence": "Table 1 reports average cost across 29 projects, ranging from $0.49 (readstat) to $1.20 (matio), with an overall average of $0.87.",
       "supported": "strong"
     },
     {
       "claim": "No significant data contamination effect exists in the benchmark (Wilcoxon p=0.27).",
       "evidence": "Table 5 compares performance on 15 pre-cutoff and 15 post-cutoff instances for GPT-4o and Claude 3 Haiku. Results show inconsistent patterns across tasks.",
       "supported": "weak"
     },
     {
       "claim": "Security tasks are significantly more challenging than general software engineering tasks, with agents scoring >60% on SWE-bench but only 18-34% on SEC-bench.",
       "evidence": "Section 3.2 compares against SWE-bench verified leaderboard scores. Claude 3.7 Sonnet achieves >60% on SWE-bench verified but 18-34% on SEC-bench tasks.",
       "supported": "weak"
     }
   ],
   "red_flags": [
     {
       "flag": "High result variance with single-run main results",
       "detail": "The one repeated experiment (SWE-agent + o3-mini) shows 30.0% ± 7.9% std dev—a coefficient of variation of 26%. Yet all main results in Table 4 are single-run point estimates. This level of variance means the reported rankings between agents/models may not be stable."
     },
     {
       "flag": "Selection and evaluation data overlap",
       "detail": "Claude 3.7 Sonnet was selected as the best model based on the 80-instance subset, then evaluated on the full 200-instance dataset that includes those same 80 instances. This inflates the reported performance for the full-dataset evaluation."
     },
     {
       "flag": "Very small contamination analysis sample",
       "detail": "The data contamination analysis uses only 30 instances (15 pre-cutoff + 15 post-cutoff) with secondary models (GPT-4o, Claude 3 Haiku), not the main evaluation model (Claude 3.7 Sonnet). The sample is too small to reliably detect contamination effects."
     },
     {
       "flag": "Unequal cost limits across models",
       "detail": "Claude 3.7 Sonnet is given a $1.5 cost limit while GPT-4o and o3-mini are capped at $1.0. Since Claude is the best-performing model, the higher budget may contribute to its advantage rather than model capability alone."
     },
     {
       "flag": "Cross-benchmark comparison is uncontrolled",
       "detail": "The claim that security tasks are harder than SE tasks is based on comparing SEC-bench scores (C/C++ memory safety, sanitizer evaluation) with SWE-bench scores (Python, test-suite evaluation). These differ in language, evaluation method, task type, and domain, making the comparison largely anecdotal."
     }
   ],
   "cited_papers": [
     {
       "title": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
       "authors": ["Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik R. Narasimhan"],
       "year": 2024,
       "relevance": "Foundational software engineering benchmark for evaluating LLM agents on real-world GitHub issues; SEC-bench positions itself as the security counterpart."
     },
     {
       "title": "SWE-agent: Agent-computer interfaces enable automated software engineering",
       "authors": ["John Yang", "Carlos E. Jimenez", "Alexander Wettig", "Kilian Lieret", "Shunyu Yao", "Karthik Narasimhan", "Ofir Press"],
       "year": 2024,
       "arxiv_id": "2405.15793",
       "relevance": "One of the three agent scaffolds evaluated in the benchmark; introduces agent-computer interface concept for code agents."
     },
     {
       "title": "OpenHands: An open platform for AI software developers as generalist agents",
       "authors": ["Xingyao Wang", "Boxuan Li", "Yufan Song"],
       "year": 2024,
       "arxiv_id": "2407.16741",
       "relevance": "Open-source agent framework evaluated in SEC-bench; SECVERIFIER is built on top of the OpenHands platform."
     },
     {
       "title": "Executable code actions elicit better LLM agents",
       "authors": ["Xingyao Wang", "Yangyi Chen", "Lifan Yuan", "Yizhe Zhang", "Yunzhu Li", "Hao Peng", "Heng Ji"],
       "year": 2024,
       "relevance": "CODEACT single-agent framework used as the baseline comparison for SECVERIFIER's multi-agent approach."
     },
     {
       "title": "Cybench: A framework for evaluating cybersecurity capabilities and risks of language models",
       "authors": ["Andy K Zhang", "Neil Perry", "Riya Dulepet"],
       "year": 2024,
       "arxiv_id": "2408.08926",
       "relevance": "CTF-based cybersecurity benchmark for LLMs; SEC-bench contrasts its automated approach against Cybench's manual CTF construction."
     },
     {
       "title": "CVE-Bench: Benchmarking LLM-based Software Engineering Agent's Ability to Repair Real-World CVE Vulnerabilities",
       "authors": ["Peiran Wang", "Xiaogeng Liu", "Chaowei Xiao"],
       "year": 2025,
       "relevance": "Vulnerability repair benchmark based on CVEFixes dataset; SEC-bench critiques its 51% label accuracy due to lack of verification."
     },
     {
       "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities",
       "authors": ["Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman"],
       "year": 2025,
       "arxiv_id": "2503.17332",
       "relevance": "Web application vulnerability benchmark for AI agents; SEC-bench contrasts its general C/C++ approach against CVE-Bench's web-framework-specific design."
     },
     {
       "title": "ARVO: Atlas of reproducible vulnerabilities for open source software",
       "authors": ["Xiang Mei", "Pulkit Singh Singaria"],
       "year": 2024,
       "arxiv_id": "2408.02153",
       "relevance": "Reproducible vulnerability dataset from OSS-FUZZ; SEC-bench extends beyond structured bug datasets to handle in-the-wild CVEs."
     },
     {
       "title": "Agentless: Demystifying LLM-based software engineering agents",
       "authors": ["Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang"],
       "year": 2024,
       "arxiv_id": "2407.01489",
       "relevance": "Two-stage agentless framework for SE tasks, representing an alternative architecture to agent-based approaches evaluated in SEC-bench."
     },
     {
       "title": "Enigma: Enhanced interactive generative model agent for CTF challenges",
       "authors": ["Talor Abramovich", "Meet Udeshi", "Minghao Shao"],
       "year": 2024,
       "arxiv_id": "2409.16165",
       "relevance": "Agent framework specifically designed for CTF security challenges, using agent-computer interfaces relevant to security task evaluation."
     },
     {
       "title": "SWE-RL: Advancing LLM reasoning via reinforcement learning on open software evolution",
       "authors": ["Yuxiang Wei", "Olivier Duchenne", "Jade Copet"],
       "year": 2025,
       "arxiv_id": "2502.18449",
       "relevance": "Applies GRPO reinforcement learning to improve agent reasoning for software engineering, a potential approach for improving security task performance."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?)",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar", "Hammond Pearce", "Ayse K. Coskun", "Gianluca Stringhini"],
       "year": 2024,
       "relevance": "Comprehensive evaluation of LLMs on vulnerability identification showing fundamental limitations, corroborating SEC-bench's findings of low agent performance."
     }
   ],
   "engagement_factors": {
     "practical_relevance": {
       "score": 2,
       "justification": "Released framework, dataset, and Docker images that security researchers and tool developers can use to evaluate LLM agents on real vulnerability tasks."
     },
     "surprise_contrarian": {
       "score": 1,
       "justification": "Low LLM performance on security tasks (18-34%) is expected and confirms the conventional view that security engineering remains beyond current agent capabilities."
     },
     "fear_safety": {
       "score": 1,
       "justification": "Touches on AI-security intersection by testing PoC generation, but results show LLMs are poor at exploit generation, which is somewhat reassuring rather than alarming."
     },
     "drama_conflict": {
       "score": 1,
       "justification": "Criticizes existing benchmarks for quality issues (up to 71% inaccurate data) but the tone is constructive and academic rather than provocative."
     },
     "demo_ability": {
       "score": 2,
       "justification": "Code on GitHub, dataset on HuggingFace, and a public leaderboard are available, though running the full benchmark requires significant Docker infrastructure."
     },
     "brand_recognition": {
       "score": 1,
       "justification": "From UIUC (Lingming Zhang's group, well-known in SE/testing) published at NeurIPS 2025, but not from a headline AI lab."
     }
   }
 }