ai-research-survey

scan.json (32517B)

---

 {
   "paper": {
     "title": "Specification-Guided Vulnerability Detection with Large Language Models",
     "authors": [
       "Hao Zhu",
       "Jia Li",
       "Cuiyun Gao",
       "Jiaru Qian",
       "Yihong Dong",
       "Huanyu Liu",
       "Lecheng Wang",
       "Ziliang Wang",
       "Xiaolong Hu",
       "Ge Li"
     ],
     "year": 2025,
     "venue": "arXiv.org",
     "arxiv_id": "2511.04014",
     "doi": "10.48550/arXiv.2511.04014"
   },
   "scan_version": 2,
   "active_modules": ["experimental_rigor", "data_leakage"],
   "methodology_tags": ["benchmark-eval", "case-study"],
   "key_findings": "VulInstruct achieves 45.0% F1-score and 37.7% recall on PrimeVul under the strict CORRECT evaluation framework, representing 32.7% and 50.8% relative improvements over the strongest baselines respectively. The approach uniquely detects 24.3% of all identified vulnerabilities (2.4x more than any baseline) through specification-guided reasoning that extracts reusable security specifications from historical vulnerabilities. Ablation shows both general and domain-specific specifications contribute complementarily, with an inverted-U relationship between knowledge quality and coverage. A real-world case study discovered CVE-2025-56538 in the Cyrus IMAP server by recognizing specification violations.",
   "checklist": {
     "artifacts": {
       "code_released": {
         "applies": true,
         "answer": true,
         "justification": "The abstract states: 'All code and supplementary materials are available at https://github.com/zhuhaopku/VulInstruct-temp.' A working URL is provided."
       },
       "data_released": {
         "applies": true,
         "answer": true,
         "justification": "The evaluation uses publicly available datasets: PrimeVul (Section 5.1) and CORRECT. The domain evidence base is constructed from publicly available NVD data (Section 4.1.2). Supplementary materials are available at the GitHub URL."
       },
       "environment_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 lists models used (Qwen3-Embedding-0.6B, DeepSeek-V3, GPT-5) and retrieval parameters (top-k=10, threshold ≥6), but provides no requirements.txt, Dockerfile, or detailed environment specifications with library versions."
       },
       "reproduction_instructions": {
         "applies": true,
         "answer": false,
         "justification": "No step-by-step reproduction instructions are described in the paper. The GitHub link is provided but no README contents, commands to run, or 'Reproducing Results' section is included in the paper itself."
       }
     },
     "statistical_methodology": {
       "confidence_intervals_or_error_bars": {
         "applies": true,
         "answer": false,
         "justification": "Tables 1, 2, and 3 report only point estimates (e.g., '45.0% F1', '37.7% recall') with no confidence intervals, error bars, or ± notation anywhere in the paper."
       },
       "significance_tests": {
         "applies": true,
         "answer": false,
         "justification": "The paper claims improvements like '32.7% relative improvement over the strongest baseline' based solely on comparing point estimates in Table 1. No statistical tests (p-values, t-tests, bootstrap, etc.) are used anywhere."
       },
       "effect_sizes_reported": {
         "applies": true,
         "answer": true,
         "justification": "Relative improvements are consistently reported with baseline context: '45.0% F1-score (32.7% improvement)', '37.7% recall (50.8% improvement)' in Section 6.1. Table 1 shows absolute values for all methods, providing context for magnitude of differences."
       },
       "sample_size_justified": {
         "applies": true,
         "answer": false,
         "justification": "The PrimeVul test set is used as-is with no justification for its size. For the threshold analysis, the authors 'randomly sample 100 out of 435 vulnerability–patch pairs' (Section 6.2) without justifying why 100 is sufficient."
       },
       "variance_reported": {
         "applies": true,
         "answer": false,
         "justification": "No standard deviations, variance, or spread measures are reported for any experimental results. All tables (1, 2, 3) show single-point results with no indication of run-to-run variation."
       }
     },
     "evaluation_design": {
       "baselines_included": {
         "applies": true,
         "answer": true,
         "justification": "Table 1 compares against 5 baselines across 4 paradigms: prompting (CoT), fine-tuning (ReVD, VulTrial), agent-based (GPTLens, VulTrial without fine-tuning), and retrieval-augmented (Vul-RAG)."
       },
       "baselines_contemporary": {
         "applies": true,
         "answer": true,
         "justification": "All baselines are from 2024-2025: ReVD (Wen et al. 2025), VulTrial (Widyasari et al. 2025), GPTLens (Hu et al. 2023, recent for this domain), Vul-RAG (Du et al. 2024). These represent the current state of the art."
       },
       "ablation_study": {
         "applies": true,
         "answer": true,
         "justification": "Table 2 presents comprehensive ablation: removing domain-specific specifications (F1 drops to 40.4%), removing general specifications (F1 drops to 41.0%), and knowledge utilization analysis showing four mutually exclusive categories. Figure 4 shows threshold ablation."
       },
       "multiple_metrics": {
         "applies": true,
         "answer": true,
         "justification": "The paper uses 7 metrics: Accuracy, Precision, Recall, F1-Score, P-C (pairwise accuracy), VP-S (pairwise discrimination), and MATCH rate (Section 5.3, Figures 5a-5b)."
       },
       "human_evaluation": {
         "applies": true,
         "answer": false,
         "justification": "The formal evaluation uses LLM-as-Judge (GPT-5) for reasoning correctness under the CORRECT framework (Section 5.4). The manual case analysis in Appendix D (20 samples) is supplementary qualitative analysis, not a formal human evaluation with structured ratings or inter-rater reliability."
       },
       "held_out_test_set": {
         "applies": true,
         "answer": true,
         "justification": "PrimeVul uses temporal data splitting: 'training data comes from vulnerabilities before a cutoff date, while test data comes from after' (Section 5.1). The knowledge base is also temporally filtered to predate the test set cutoff."
       },
       "per_category_breakdown": {
         "applies": true,
         "answer": true,
         "justification": "Figure 5a provides a radar chart comparing MATCH rates across 8 CWE categories. Figure 5b shows head vs. tail CWE performance. Table 2 breaks down performance by knowledge source type."
       },
       "failure_cases_discussed": {
         "applies": true,
         "answer": true,
         "justification": "Appendix C provides detailed failure analysis of Vul-RAG (premature secure classification, Table 5a) and ReVD (categorized failure types including zero reasoning, wrong type, over-generalization, Table 5b). Section 6.2 discusses the 0.7% of cases with no matched knowledge yielding 0% F1."
       },
       "negative_results_reported": {
         "applies": true,
         "answer": true,
         "justification": "The threshold analysis (Figure 4) shows that both too-lenient (3,3,3) and too-strict (9,9,9) thresholds hurt performance significantly. VulInstruct's precision (55.8%) is lower than fine-tuned VulTrial's (59.9%) in Table 1. Ablation shows domain-specific-only knowledge yields 0% F1."
       }
     },
     "claims_and_evidence": {
       "abstract_claims_supported": {
         "applies": true,
         "answer": true,
         "justification": "Abstract claims of 45.0% F1 (32.7% improvement), 37.7% recall (50.8% improvement), 24.3% unique detections (2.4x), and 32.3% pairwise improvement all match Table 1. The CVE-2025-56538 discovery is detailed in Appendix E."
       },
       "causal_claims_justified": {
         "applies": true,
         "answer": true,
         "justification": "The primary causal claims ('specifications improve detection') are supported by controlled ablation studies in Table 2: removing domain-specific specifications reduces F1 by 4.6%, removing general specifications reduces it by 4.0%. These are single-variable manipulations adequate for the causal claims."
       },
       "generalization_bounded": {
         "applies": true,
         "answer": false,
         "justification": "The title 'Specification-Guided Vulnerability Detection with Large Language Models' and claims like 'establishes new state-of-the-art' are broad, but results are exclusively on PrimeVul (C/C++ function-level detection). The paper does not explicitly bound claims to C/C++ or function-level analysis."
       },
       "alternative_explanations_discussed": {
         "applies": true,
         "answer": false,
         "justification": "Section 8 discusses implementation validity, experimental fairness, temporal validity, and LLM-as-Judge reliability, but does not consider alternative explanations for VulInstruct's improvements (e.g., whether providing any additional context rather than specifically specifications would help equally, or whether the retrieval mechanism matters more than specification quality)."
       },
       "proxy_outcome_distinction": {
         "applies": true,
         "answer": true,
         "justification": "The paper explicitly argues that binary classification is an inadequate proxy for vulnerability understanding (Section 2.1, 5.3) and adopts CORRECT evaluation requiring correct reasoning, not just correct labels. Claims are made at the granularity of the metrics (F1 under CORRECT, MATCH rate)."
       }
     },
     "setup_transparency": {
       "model_versions_specified": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 and Tables 1/3 list models as 'DeepSeek-V3', 'GPT-5', 'GPT-OSS-120B', 'Claude-Sonnet-4', 'DeepSeek-R1', 'Qwen3-Embedding-0.6B'. None include snapshot dates or API versions. Per the schema, marketing names without snapshot dates do not count."
       },
       "prompts_provided": {
         "applies": true,
         "answer": true,
         "justification": "Appendix B (Section B) provides the full text of all key prompts: general specification extraction, detailed vulnerability case generation, knowledge scoring, domain-specific specification extraction, and vulnerability detection prompts, including format specifications and one-shot examples."
       },
       "hyperparameters_reported": {
         "applies": true,
         "answer": false,
         "justification": "Section 5.4 reports retrieval parameters (top-k=10, threshold ≥6) and embedding model, but does not report LLM generation parameters (temperature, top-p, max tokens) for any of the models used in the pipeline."
       },
       "scaffolding_described": {
         "applies": true,
         "answer": true,
         "justification": "The multi-stage pipeline is described in detail in Section 4 with Figure 3: offline knowledge construction, code context extraction (Eq. 2), dual-path retrieval, knowledge scoring, domain-specific specification generation, and structured reasoning. Appendix F describes the automatic context extraction tool."
       },
       "data_preprocessing_documented": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 documents NVD crawling (173,070 valid cases, 15,391 rejected, 2002-2024). Section 5.1 documents temporal filtering (1,338 pairs from CORRECT predating PrimeVul test cutoff). Section 4.1.1 describes the context extraction pipeline."
       }
     },
     "limitations_and_scope": {
       "limitations_section_present": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 'Threats to Validity' provides substantive discussion across four areas: implementation validity, experimental fairness, temporal validity, and LLM-as-Judge reliability. Section 7 'Discussion' also discusses directions for improvement."
       },
       "threats_to_validity_specific": {
         "applies": true,
         "answer": true,
         "justification": "Section 8 discusses study-specific threats: baseline failure analysis (Vul-RAG premature classification, ReVD reasoning failures), the choice to exclude CWE information from all methods for fairness, exclusion of SVEN dataset due to lacking CVE metadata, and the specific bias risk of using GPT-5 as judge."
       },
       "scope_boundaries_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not explicitly state what results do NOT show. It doesn't bound findings to C/C++ function-level detection, doesn't discuss limitations to the PrimeVul vulnerability types, and doesn't acknowledge that real-world deployment differs from benchmark evaluation. Section 7 discusses future work but not scope boundaries."
       }
     },
     "data_integrity": {
       "raw_data_available": {
         "applies": true,
         "answer": false,
         "justification": "While the GitHub URL is provided and input datasets are public (PrimeVul, NVD), the paper does not explicitly state that raw experimental outputs (per-example predictions, LLM responses, scoring details) are available for independent verification."
       },
       "data_collection_described": {
         "applies": true,
         "answer": true,
         "justification": "Section 4.1.2 describes NVD data collection (crawling, normalization, temporal range 2002-2024, fields extracted). Section 5.1 describes CORRECT and PrimeVul dataset roles. Section 4.1.1 describes context extraction with Eq. 2."
       },
       "recruitment_methods_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants. Data sources are standard benchmarks (PrimeVul, CORRECT) and public databases (NVD)."
       },
       "data_pipeline_documented": {
         "applies": true,
         "answer": true,
         "justification": "The full pipeline is documented: NVD crawling → normalization → temporal filtering (Section 4.1.2), CORRECT dataset → context extraction → specification extraction (Section 4.1.1), PrimeVul test set with temporal cutoff (Section 5.1). Figure 3 provides a visual overview."
       }
     },
     "conflicts_of_interest": {
       "funding_disclosed": {
         "applies": true,
         "answer": false,
         "justification": "No funding sources, grants, or acknowledgments section is present anywhere in the paper."
       },
       "affiliations_disclosed": {
         "applies": true,
         "answer": true,
         "justification": "All author affiliations are listed on the first page: Peking University, Tsinghua University, Harbin Institute of Technology, and New H3C Technologies Co., Ltd."
       },
       "funder_independent_of_outcome": {
         "applies": true,
         "answer": false,
         "justification": "No funding is disclosed, so independence cannot be assessed. One author (Xiaolong Hu) is affiliated with New H3C Technologies Co., Ltd, a commercial entity, but no funding relationship is stated."
       },
       "financial_interests_declared": {
         "applies": true,
         "answer": false,
         "justification": "No competing interests or financial interests statement is present in the paper."
       }
     },
     "contamination": {
       "training_cutoff_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not state the training data cutoff dates for any of the LLMs used (DeepSeek-V3, GPT-5, DeepSeek-R1, Claude-Sonnet-4). PrimeVul's temporal split is discussed but this concerns the knowledge base, not the LLM pre-training data."
       },
       "train_test_overlap_discussed": {
         "applies": true,
         "answer": false,
         "justification": "The paper discusses temporal filtering for the knowledge base (Section 5.1, Section 8) but does not discuss whether the LLMs (DeepSeek-V3, GPT-5, etc.) may have been pre-trained on PrimeVul test data or the CORRECT vulnerability descriptions used in evaluation."
       },
       "benchmark_contamination_addressed": {
         "applies": true,
         "answer": false,
         "justification": "PrimeVul was published in 2024. The LLMs used (DeepSeek-V3, GPT-5) could have been trained on PrimeVul data or the underlying CVE descriptions. The paper relies on PrimeVul's temporal split but does not address LLM pre-training contamination with the benchmark itself."
       }
     },
     "human_studies": {
       "pre_registered": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "irb_or_ethics_approval": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "demographics_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "inclusion_exclusion_criteria": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "randomization_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "blinding_described": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       },
       "attrition_reported": {
         "applies": false,
         "answer": false,
         "justification": "No human participants in this study."
       }
     },
     "cost_and_practicality": {
       "inference_cost_reported": {
         "applies": true,
         "answer": false,
         "justification": "VulInstruct involves multiple LLM calls per example (query generation, knowledge scoring for up to 20 items, domain-specific specification generation, and final detection), but no inference cost, latency, or tokens consumed are reported."
       },
       "compute_budget_stated": {
         "applies": true,
         "answer": false,
         "justification": "No total computational budget is stated. No API costs, GPU hours, or total processing time reported for the 435 test pairs or knowledge base construction."
       }
     },
     "experimental_rigor": {
       "seed_sensitivity_reported": {
         "applies": true,
         "answer": false,
         "justification": "No mention of multiple random seeds. All results in Tables 1, 2, and 3 appear to be from single experimental runs with no seed sensitivity analysis."
       },
       "number_of_runs_stated": {
         "applies": true,
         "answer": false,
         "justification": "The paper never explicitly states how many experimental runs produced the reported results. No 'averaged over K runs' or equivalent statement."
       },
       "hyperparameter_search_budget": {
         "applies": true,
         "answer": false,
         "justification": "Figure 4 shows a threshold analysis with 4 settings, but the total search budget for other hyperparameters (top-k, embedding model choice, scoring rubric design) is not reported. The threshold analysis was conducted on 100 test samples, not a validation set."
       },
       "best_config_selection_justified": {
         "applies": true,
         "answer": false,
         "justification": "The threshold analysis (Figure 4) was performed on 100 randomly sampled test pairs. Tuning hyperparameters on test data compromises the evaluation. Additionally, Figure 4 shows (5,5,5) as optimal but Section 5.4 uses (6,6,6) without explanation for the discrepancy."
       },
       "multiple_comparison_correction": {
         "applies": false,
         "answer": false,
         "justification": "No statistical tests are performed anywhere in the paper, so multiple comparison correction is not applicable."
       },
       "self_comparison_bias_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The authors implement and evaluate their own system against baselines without acknowledging potential bias from evaluating their own system. While they analyze baseline failures (Appendix C), they don't acknowledge that their implementations or configurations of baselines might underperform."
       },
       "compute_budget_vs_performance": {
         "applies": true,
         "answer": false,
         "justification": "VulInstruct requires multiple LLM calls per example (retrieval queries, scoring each of ~20 retrieved items, specification generation, detection), while CoT baselines use a single call. This significant compute difference is never discussed or compared against performance gains."
       },
       "benchmark_construct_validity": {
         "applies": true,
         "answer": true,
         "justification": "The paper extensively discusses benchmark validity: Section 2.1 argues that binary classification is insufficient, Section 5.3 explains why CORRECT evaluation is needed to assess genuine vulnerability understanding, and Section 8 discusses why SVEN was excluded. The adoption of reasoning-aware evaluation directly addresses construct validity."
       },
       "scaffold_confound_addressed": {
         "applies": true,
         "answer": false,
         "justification": "Table 1 compares VulInstruct (DeepSeek-V3) against ReVD (fine-tuned Qwen2.5-Coder), VulTrial (fine-tuned GPT-4o), and GPTLens (DeepSeek-V3), conflating model and method effects. While Table 3 partially addresses this by testing VulInstruct across multiple models, the main comparison does not control for the scaffolding confound."
       }
     },
     "data_leakage": {
       "temporal_leakage_addressed": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 enforces strict temporal filtering: 'We use PrimeVul's test set cutoff date as our boundary. Only vulnerabilities published before this date are included in our knowledge base.' Section 8 further confirms CVE retrieval uses temporal and ID-ordering constraints."
       },
       "feature_leakage_addressed": {
         "applies": true,
         "answer": false,
         "justification": "While temporal filtering prevents using future data, the paper does not discuss whether retrieved specifications or CVE cases could encode information about the specific test vulnerability (e.g., if a specification was extracted from a closely related vulnerability in the same project)."
       },
       "non_independence_addressed": {
         "applies": true,
         "answer": false,
         "justification": "The paper does not discuss whether the knowledge base (from CORRECT) and test set (from PrimeVul) share vulnerabilities from the same projects, codebases, or authors, which could create non-independence between training and test conditions."
       },
       "leakage_detection_method": {
         "applies": true,
         "answer": true,
         "justification": "Section 5.1 applies temporal filtering as a concrete prevention method: knowledge base entries must predate PrimeVul's test cutoff. Section 4.2 additionally filters domain-specific CVEs by publication date and CVE ID ordering relative to the target."
       }
     }
   },
   "claims": [
     {
       "claim": "VulInstruct achieves 45.0% F1-score, a 32.7% relative improvement over the strongest baseline (GPTLens at 33.9%) on PrimeVul.",
       "evidence": "Table 1 shows F1 scores for all methods. GPTLens achieves 33.9%, VulInstruct 45.0%. Section 6.1 discusses the results.",
       "supported": "moderate"
     },
     {
       "claim": "VulInstruct achieves 37.7% recall, a 50.8% improvement over GPTLens (25.0%), detecting substantially more vulnerabilities.",
       "evidence": "Table 1 shows recall values. VulInstruct's recall is highest at 37.7% vs GPTLens at 25.0%. Section 6.1.",
       "supported": "moderate"
     },
     {
       "claim": "VulInstruct uniquely detects 24.3% of all identified vulnerabilities, 2.4x more than any baseline.",
       "evidence": "Table 1 'Unique' column shows VulInstruct at 24.3% vs next best VulTrial at 10.2%. Section 6.1 defines Unique as percentage of vulnerabilities exclusively detected by one method among all 226 detected CVEs.",
       "supported": "moderate"
     },
     {
       "claim": "Both general and domain-specific specifications are complementary and contribute to detection performance.",
       "evidence": "Table 2 ablation: removing domain-specific specs drops F1 from 45.0% to 40.4% (-4.6%); removing general specs drops to 41.0% (-4.0%). Utilization analysis shows 73.5% of samples use both sources, achieving 48.3% F1.",
       "supported": "strong"
     },
     {
       "claim": "VulInstruct generalizes effectively across CWE types and different base LLMs.",
       "evidence": "Figure 5a shows consistently higher MATCH rates across 8 CWE categories. Figure 5b shows improvements on both head (38.4%) and tail (28.8%) CWEs. Table 3 shows improvements on GPT-OSS-120B, Claude-Sonnet-4, and DeepSeek-R1.",
       "supported": "moderate"
     },
     {
       "claim": "An optimal threshold exists for balancing knowledge quality and coverage (inverted-U relationship).",
       "evidence": "Figure 4 shows F1 peaks at (5,5,5) threshold (43.4%) and drops at both lenient (3,3,3: 40.2%) and strict (9,9,9: 31.1%) settings. Section 6.2.",
       "supported": "moderate"
     },
     {
       "claim": "VulInstruct discovered a previously unknown high-severity vulnerability (CVE-2025-56538) in the Cyrus IMAP server.",
       "evidence": "Section 7 and Appendix E detail the case study: a privilege bypass in mboxlist_renamemailbox via a goto statement bypassing ACL checks. The vulnerability was confirmed by developers, reproduced via integration tests, and patched.",
       "supported": "strong"
     }
   ],
   "red_flags": [
     {
       "flag": "No uncertainty quantification",
       "detail": "All results across Tables 1, 2, and 3 are single-point estimates with no confidence intervals, error bars, variance, or statistical tests. For LLM-based systems with inherent stochasticity, this is a significant omission that makes it impossible to assess result stability."
     },
     {
       "flag": "LLM-as-Judge circular dependency",
       "detail": "The CORRECT evaluation framework uses GPT-5 to judge reasoning correctness. Using an LLM to evaluate other LLMs' vulnerability reasoning introduces potential bias. The paper acknowledges this in Section 8 but only notes they use a different model from those evaluated."
     },
     {
       "flag": "Threshold tuned on test data",
       "detail": "The threshold analysis (Figure 4) randomly samples 100 of 435 test pairs to select the optimal threshold. Tuning hyperparameters on test data compromises evaluation integrity. Additionally, the analysis finds (5,5,5) optimal but Section 5.4 uses (6,6,6) without explaining the discrepancy."
     },
     {
       "flag": "Undisclosed compute costs",
       "detail": "VulInstruct requires multiple LLM calls per example (query generation, scoring ~20 retrieved items, specification generation, detection), making it substantially more expensive than single-call baselines. This cost is never quantified or discussed."
     },
     {
       "flag": "LLM pre-training contamination unaddressed",
       "detail": "While the knowledge base is temporally filtered, the underlying LLMs (DeepSeek-V3, GPT-5, etc.) may have been pre-trained on PrimeVul data, CVE descriptions, or patches used in evaluation. This contamination vector is not discussed."
     },
     {
       "flag": "Single real-world case study presented as evidence of practical utility",
       "detail": "The discovery of CVE-2025-56538 (Appendix E) is compelling but represents a single case using a different LLM (Gemini-2.5-Pro) than the main experiments. Generalizing from one case to 'practical value for real-world vulnerability discovery' overstates the evidence."
     }
   ],
   "cited_papers": [
     {
       "title": "Vulnerability Detection with Code Language Models: How Far Are We?",
       "authors": ["Yangruibo Ding", "Yanjun Fu", "Omniyyah Ibrahim"],
       "year": 2024,
       "arxiv_id": "2403.18624",
       "relevance": "Introduces the PrimeVul benchmark used for evaluation and documents that LLMs achieve <12% accuracy distinguishing vulnerable from patched code."
     },
     {
       "title": "Everything You Wanted to Know About LLM-based Vulnerability Detection But Were Afraid to Ask",
       "authors": ["Yue Li", "Xiao Li", "Hao Wu"],
       "year": 2025,
       "arxiv_id": "2504.13474",
       "relevance": "Proposes the CORRECT evaluation framework requiring both correct predictions and valid reasoning, adopted as the primary evaluation methodology."
     },
     {
       "title": "Large Language Model-Powered Smart Contract Vulnerability Detection: New Perspectives",
       "authors": ["Sihao Hu", "Tiansheng Huang", "Fatih İlhan"],
       "year": 2023,
       "relevance": "Proposes GPTLens multi-agent framework for vulnerability detection, used as a key baseline achieving the second-best F1 in comparisons."
     },
     {
       "title": "Let the Trial Begin: A Mock-Court Approach to Vulnerability Detection using LLM-Based Agents",
       "authors": ["Ratnadira Widyasari", "Martin Weyssow"],
       "year": 2025,
       "arxiv_id": "2505.10961",
       "relevance": "Introduces VulTrial multi-agent courtroom-inspired vulnerability detection framework, a key baseline with 102.39% improvement over single-agent approaches."
     },
     {
       "title": "Boosting Vulnerability Detection of LLMs via Curriculum Preference Optimization with Synthetic Reasoning Data",
       "authors": ["Xin-Cheng Wen", "Yijun Yang", "Cuiyun Gao"],
       "year": 2025,
       "doi": "10.18653/v1/2025.findings-acl.467",
       "relevance": "Proposes ReVD using reinforcement learning with reasoning distillation for vulnerability detection, a fine-tuning baseline."
     },
     {
       "title": "Vul-RAG: Enhancing LLM-based Vulnerability Detection via Knowledge-Level RAG",
       "authors": ["Xueying Du", "Geng Zheng", "Kaixin Wang"],
       "year": 2024,
       "relevance": "RAG-based vulnerability detection approach used as a baseline; its retrieval-based decision mechanism is analyzed in the failure study."
     },
     {
       "title": "LLMs Cannot Reliably Identify and Reason About Security Vulnerabilities (Yet?)",
       "authors": ["Saad Ullah", "Mingji Han", "Saurabh Pujar"],
       "year": 2024,
       "arxiv_id": "2312.12575",
       "relevance": "Comprehensive evaluation showing fundamental limitations of LLMs in vulnerability reasoning, motivating the specification-guided approach."
     },
     {
       "title": "Top Score on the Wrong Exam: On Benchmarking in Machine Learning for Vulnerability Detection",
       "authors": ["Niklas Risse", "Jing Liu", "Marcel Böhme"],
       "year": 2025,
       "relevance": "Reports that most functions in vulnerability benchmarks cannot be classified without knowing invocation context, motivating richer context extraction."
     },
     {
       "title": "Harnessing Large Language Models for Software Vulnerability Detection: A Comprehensive Benchmarking Study",
       "authors": ["Karl Tamberg", "Hayretdin Bahsi"],
       "year": 2025,
       "doi": "10.1109/access.2025.3541146",
       "relevance": "Comprehensive benchmarking study of LLMs for vulnerability detection providing context for the state of the field."
     },
     {
       "title": "R2Vul: Learning to Reason about Software Vulnerabilities with Reinforcement Learning and Structured Reasoning Distillation",
       "authors": ["Martin Weyssow", "Chengran Yang"],
       "year": 2025,
       "arxiv_id": "2504.04699",
       "relevance": "Combines reinforcement learning with structured reasoning distillation for vulnerability detection with explanations."
     },
     {
       "title": "Benchmarking LLMs and LLM-based Agents in Practical Vulnerability Detection for Code Repositories",
       "authors": ["Alperen Yildiz", "Sin G Teo"],
       "year": 2025,
       "arxiv_id": "2503.03586",
       "relevance": "Benchmarks LLM-based agents for repository-level vulnerability detection, exploring ReAct agents for recursive callee retrieval."
     },
     {
       "title": "BugScope: Learn to Find Bugs Like Human",
       "authors": ["Jinyao Guo", "Chengpeng Wang"],
       "year": 2025,
       "relevance": "Uses LLMs to summarize vulnerability cases and expand code patterns for vulnerability types, a related knowledge augmentation approach."
     }
   ]
 }